{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai import *\n",
    "from fastai.text import *\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "from fastai.callbacks.tracker import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "dir_pth = Path('./')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_pth = dir_pth / 'train.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_pth = dir_pth / 'test.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = pd.read_csv(train_pth)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>keyword</th>\n",
       "      <th>location</th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Our Deeds are the Reason of this #earthquake M...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Forest fire near La Ronge Sask. Canada</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>All residents asked to 'shelter in place' are ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>13,000 people receive #wildfires evacuation or...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Just got sent this photo from Ruby #Alaska as ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id keyword location                                               text  \\\n",
       "0   1     NaN      NaN  Our Deeds are the Reason of this #earthquake M...   \n",
       "1   4     NaN      NaN             Forest fire near La Ronge Sask. Canada   \n",
       "2   5     NaN      NaN  All residents asked to 'shelter in place' are ...   \n",
       "3   6     NaN      NaN  13,000 people receive #wildfires evacuation or...   \n",
       "4   7     NaN      NaN  Just got sent this photo from Ruby #Alaska as ...   \n",
       "\n",
       "   target  \n",
       "0       1  \n",
       "1       1  \n",
       "2       1  \n",
       "3       1  \n",
       "4       1  "
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data = pd.read_csv(test_pth)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>keyword</th>\n",
       "      <th>location</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Just happened a terrible car crash</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Heard about #earthquake is different cities, s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>there is a forest fire at spot pond, geese are...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Apocalypse lighting. #Spokane #wildfires</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>11</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Typhoon Soudelor kills 28 in China and Taiwan</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id keyword location                                               text\n",
       "0   0     NaN      NaN                 Just happened a terrible car crash\n",
       "1   2     NaN      NaN  Heard about #earthquake is different cities, s...\n",
       "2   3     NaN      NaN  there is a forest fire at spot pond, geese are...\n",
       "3   9     NaN      NaN           Apocalypse lighting. #Spokane #wildfires\n",
       "4  11     NaN      NaN      Typhoon Soudelor kills 28 in China and Taiwan"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([nan, 'ablaze', 'accident', 'aftershock', ..., 'wounds', 'wreck', 'wreckage', 'wrecked'], dtype=object)"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.keyword.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([nan, 'Birmingham', 'Est. September 2012 - Bristol', 'AFRICA', ..., '#NewcastleuponTyne #UK',\n",
       "       'Vancouver, Canada', 'London ', 'Lincoln'], dtype=object)"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.location.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 0])"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.target.unique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 只用text来进行判断尝试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "text = pd.concat([train_data[['id', 'text']], test_data[['id', 'text']]], axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Our Deeds are the Reason of this #earthquake M...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>Forest fire near La Ronge Sask. Canada</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>All residents asked to 'shelter in place' are ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>13,000 people receive #wildfires evacuation or...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7</td>\n",
       "      <td>Just got sent this photo from Ruby #Alaska as ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id                                               text\n",
       "0   1  Our Deeds are the Reason of this #earthquake M...\n",
       "1   4             Forest fire near La Ronge Sask. Canada\n",
       "2   5  All residents asked to 'shelter in place' are ...\n",
       "3   6  13,000 people receive #wildfires evacuation or...\n",
       "4   7  Just got sent this photo from Ruby #Alaska as ..."
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert len(text) == len(train_data) + len(test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "text.to_csv('text.csv', index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data_lm = TextLMDataBunch.from_csv(dir_pth, 'text.csv', text_cols=1, label_cols=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = language_model_learner(data_lm, AWD_LSTM, drop_mult=1)\n",
    "learn.callbacks.append(EarlyStoppingCallback(learn, patience = 10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n",
      "Min numerical gradient: 5.75E-02\n",
      "Min loss divided by 10: 5.25E-02\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3dd3hc5Zn38e+t0ag3F7nbuICNgdgGC9PbkkBI5wpsQiihBMcbQkjIsmE3b3azSdgNSwoQNrAOgQBLqklCQg9kCQSwiYxlG2yDe5VtSbZQL6O53z9mBEKWZdnWmaL5fa5rLs2c88w5t0ajuecp53nM3RERkcyVlewAREQkuZQIREQynBKBiEiGUyIQEclwSgQiIhkuO9kBHKyRI0f65MmTkx2GiEhaWbp0aa27l/e1L+0SweTJk6msrEx2GCIiacXMNu9vn5qGREQynBKBiEiGUyIQEclwSgQiIhlOiUBEJMMFmgjMrMzMFpnZGjNbbWan9FHmbDOrMrM3zOwvQcYjIiL7Cnr46B3AU+5+kZnlAAU9d5pZGfBj4IPuvsXMRgUcj4iI9BJYjcDMSoEzgZ8CuHuHu9f3KvYZ4LfuviVeZndQ8YiIpLPbn32LF9fWBHLsIJuGpgA1wP1mtszM7jWzwl5lpgPDzOx5M1tqZlf0dSAzm29mlWZWWVMTzAshIpKq3J0f/XkdSzbsCeT4QSaCbOAE4G53Px5oBm7uo8xc4MPA+cA3zGx67wO5+0J3r3D3ivLyPq+QFhEZsto6o3RFnaK8YFrzg0wE24Bt7r4k/ngRscTQu8zT7t7s7rXAC8DsAGMSEUk7je2dABTmplkicPedwFYzmxHfdC6wqlexR4HTzSzbzAqAk4DVQcUkIpKOmtoiABQHlAiCHjV0PfBwfMTQBuAqM1sA4O73uPtqM3sKWAFEgXvd/fWAYxIRSSvN7V0AFKVjInD3KqCi1+Z7epW5DbgtyDhERNJZd9NQOvYRiIjIIOhuGgqqRqBEICKS4pra430EqhGIiGSm7kSQdqOGRERkcDSqaUhEJLM1t0cIh4zc7GA+spUIRERSXFN7hKLcbMwskOMrEYiIpLimtkhgQ0dBiUBEJOU1tkcoyg0HdnwlAhGRFNfUFqEoNxTY8ZUIRERSXHcfQVCUCEREUlxTe4SiPDUNiYhkLNUIREQyXFNbJLDpJUCJQEQkpUW6orR2dqlGICKSqbrXIghqniFQIhARSWndaxEEtToZKBGIiKS07plHdWWxiEiGam4PduZRUCIQEUlp70xBrRqBiEhmemd1MtUIREQyU/d6xRo1JCKSodK+s9jMysxskZmtMbPVZnbKfsqdaGYRM7soyHhERNJNdx9BYU5wiSC4I8fcATzl7heZWQ5Q0LuAmYWAW4FnAo5FRCTtNLdHKMwJEcoKZnUyCLBGYGalwJnATwHcvcPd6/soej3wCLA7qFhERNJVbObRYL+zB9k0NAWoAe43s2Vmdq+ZFfYsYGbjgQuBu/s7kJnNN7NKM6usqakJLmIRkRTTGPDMoxBsIsgGTgDudvfjgWbg5l5lbge+5u7R/g7k7gvdvcLdK8rLy4OJVkQkBcXWKw5uLQIIto9gG7DN3ZfEHy9i30RQAfzSzABGAh8ys4i7/z7AuERE0kZsLYLglqmEABOBu+80s61mNsPd3wTOBVb1KjOl+76Z/Qx4TElARORdTW0RRhbtM85mUAU9auh64OH4iKENwFVmtgDA3e8J+NwiImkvViNI36Yh3L2KWPNPT30mAHe/MshYRETSUVN7sKuTga4sFhFJWe4e+HrFoEQgIpKy2jqjdEU9ra8jEBGRw9C9OlmQE86BEoGISMrqnnk0yCmoQYlARCRlNSVgdTJQIhARSVmJmIIalAhERFJWd9OQagQiIhnqnWUqVSMQEclM3YlAo4ZERDJUo5qGREQyW1N7hHDIyM0O9qNaiUBEJEU1x6eXiE/VHxglAhGRFBVblCboSaKVCEREUlZjAqagBiUCEZGU1dQW/OpkoEQgIpKyEjEFNSgRiIikrKb24BeuByUCEZGUpRqBiEiGa2oLfplKUCIQEUlJka4orZ1dqhGIiGSq5vYuIPh5hiDgRGBmZWa2yMzWmNlqMzul1/5LzWyFma00s5fNbHaQ8YiIpIvuZSqDXp0MIOgz3AE85e4XmVkOUNBr/0bgLHffa2YXAAuBkwKOSUQk5SVqURoIMBGYWSlwJnAlgLt3AB09y7j7yz0eLgYmBBWPiEg6aU7QMpUQbNPQFKAGuN/MlpnZvWZW2E/5a4An+9phZvPNrNLMKmtqaoKIVUQkpbwzBXWajxrKBk4A7nb344Fm4Oa+CprZOcQSwdf62u/uC929wt0rysvLg4pXRCRlvLM6WZrXCLYB29x9SfzxImKJ4T3MbBZwL/Bxd68LMB4RkbTRvV5xWo8acvedwFYzmxHfdC6wqmcZM5sE/Ba43N3fCioWEZF0MyQ6i+OuBx6OjxjaAFxlZgsA3P0e4F+BEcCP4wsvRNy9IuCYRERSXncfQWFOmicCd68Cen+w39Nj/+eAzwUZg4hIOmpqj1CYEyKUFezqZKAri0VEUlJDayfFCZh5FJQIRERSUl1zByOKchJyLiUCEZEUVNfUzoii3IScS4lARCQF1TZ1MLJQNQIRkYzk7tQ1t6tpSEQkU7V0dNHWGVXTkIhIpqpris3POVKJQEQkM9U2twOoaUhEJFO9UyMoVI1ARCQj1TWpRiAiktHqmmM1guEaPioikplqm9opzs0mLxxKyPmUCEREUkxdU+KmlwAlAhGRlBO7mCwxHcWgRCAiknLqmjoYkaD+AVAiEBFJObUJnHAOlAhERFJKV9TZ09zBSPURiIhkpvqWDqKOmoZERDJV9zUEahoSEclQtQm+qhiUCEREUkqiZx4FJQIRkZTSPc/QkEkEZlZmZovMbI2ZrTazU3rtNzO708zWmdkKMzshyHhERFJdXXMHWQZl+eGEnTM74OPfATzl7heZWQ5Q0Gv/BcBR8dtJwN3xnyIiGam2qYPhhblkZVnCzjmgGoGZTTOz3Pj9s83sS2ZWdoDnlAJnAj8FcPcOd6/vVezjwIMesxgoM7OxB/1biIgMEXVN7Qm9hgAG3jT0CNBlZkcCC4GJwM8P8JwpQA1wv5ktM7N7zaywV5nxwNYej7fFt72Hmc03s0ozq6ypqRlgyCIi6aeuObETzsHAE0HU3SPAhcCP3P0m4EDf3LOBE4C73f14oBm4+VCCdPeF7l7h7hXl5eWHcggRkbRQ19TOiAStTNZtoImg08wuAT4LPBbfdqCejG3ANndfEn+8iFhi6Gk7sdpFtwnxbSIiGSnRU1DDwBPBVcApwC3uvtHMpgAP9fcEd98JbDWzGfFN5wKrehX7A3BFfPTQycDb7l498PBFRIaOts4uGtsjCR06CgMcNeTuq4AvAZjZMKDY3W8dwFOvBx6OjxjaAFxlZgvix7wHeAL4ELAOaCGWcAK1qbaZZ1fv4rnVu9nV2EZpfpiy/DBlBTlMGJbPlJGFTB5ZyLTyIkoTOHwrk3VFnaqte/nTqt0s31rPzLElnHbkCE6aOoKi3KAHtomkjj3d00skcJ4hGGAiMLPngY/Fyy8FdpvZS+5+Y3/Pc/cqoKLX5nt67HfguoMJ+FC98FYN33psFet2NwEwY3QxR48ppqE1Qk1TO2/tauLRqlaiHiufZVBxxHDOO3Y05x0zhkkjeo98lUPxyvo6nlm1k5b2Llo7u2jpiPDalnr2NHeQnWXMGFPMw0s2c99LGwllGUeWFzG8MIdhhbFkHc4yHIi6k52VxZjSPMaV5TO+LI/yojxK8rMpzgsTyjLaOrvY1dBG9dtt7GnuoD3SRUckSnskSltnF83tsfN3djmjSnKZMKyA8WX5jCrOJTc7i9zsELnhrIQtFyjy7vQSKVgjAErdvcHMPkdsuOe/mdmKIAMbbGUFYUaX5HLpSZN4/8zRTBy+7wd7RyTK1r0tbKptZvnWep5ZtYvvPL6a7zy+mpljS/jo7LF8dNa4Pp+bCtydh5ds4eElW8gPZ1FWkENZQZgRhTmMLsljTGkeo4rzyAtnkWVGKMsoyAkxviyf7FDWPsdq6egiPxwalPHM62ua+M8nVvPs6t3kh0OU5GdTkBNbk/XMo0Zy7szRnDm9nNL8MG2dXby2eS8vra/lrV1N1Ld08ObORupbOolEnSwDM6MzEqWxPbLPucygIByiuaPrgHHlh0Nkh4zGtn2P023i8HxOPGI4cycPY+4Rw5hWXkQ4pIvyZfB1Ty+R6D4Ci30pP0Ahs5XAecADwNfd/W9mtsLdZwUdYG8VFRVeWVmZsPNt3dPC02/s5PGV1SzbErsMYtaEUk6eOoI5E8uYM7GMsaV5dHY5ze0RmjsiRKOx55pBe6SLjbUtrK9pYv3uJvY0dxCJOpFolK6oU5wX+6AeVpjDxGEFnHfs6H3aB92dbXtbaY9EcXeiDiX52YwpycMs9iG9o76Vrz2yghfX1jJ7Qikl+WH2tnSwt7mT2qZ22iPR/f6O4ZAxaXgBU8uLiEZj59q2t4Xmji5CWcawgjDDC3PIzQ7R2NZJQ1uEpvYIJXnZjCvLZ2xpHmNL8xlVksuo4jxGFeeSZUZtUzu1Te28tauR3762nbxwiH84exrXnD5l0L5lN7VHqK5vZVt9K3VNHbzd2snbrZ00tUUYVhBmTDy2EUU55IVD5GZnkZOdRX449J4k19weYXt97Peubep4p+bQ0h7hjR0NVG7eQ238nzQnlMXU8kKOHlPMjDElzBxbzDFjSygvziXqsW911W+34e68b3zpPklWZH9+U7mVmxat4IWbzhn0VggzW+ruvVtoYvsGmAguBr4BvOTu/2BmU4Hb3P2TgxrpACQ6EfS0dU8Lj6+s5uk3dvLG9gY6umIfrtlZRiR64NdxVHEu5cW5ZIeyCGcZWWY0tHVS19zB3niCCGUZZ00v5xPHxy6neOGtGl5cW8OuhvZ9jldenMuciWVMHVnIz5dsocudf/nQTC49adI7CQJiieTt1k52NbSzq6GNjkiULneiUaexLcKG2mY21DSxobaZ7CxjwrACJgzLZ3RJHs3tEfa0dLCnKda0UpwXpjgvm6K8bBpaO9le30Z1fSvVb7fR1Me3c4Cc7CwunjuBr3xgesI7wQaLu7OproWqrXtZs7ORN+O36rfb3ilTnJdNa0fXe94LwwrCnHP0KN4/czSjS/Joao/Q3B6hsyvK7AllHDGi4D1/K8ls9/xlPd99cg1v/Pv5FA5y/9hhJ4JUksxE0FN7pIvV1Y1UbdnLzoZ2inJDFOZmU5ATIjsri+5XNRwyjhhRyNTyQkry9t/57O68uauR3y/bwaNV29/5gCnND3P6USM5ZeoISvLDsWYRYt+2q7bWU7W1no21zcybPJzbLp7FESN6X7OXOC0dEWoa29nd2E406owszmVkYS4l+dlD9sOuvqWDNTsbWV3dwPqaJkrzw4wpzWdcaR6tnV38efVu/vzmbupbOvt8/tjSPE6ZOoIzp5fz/mNGq3M8w93y+CoeWryZ1d/64KD/zwxGjWAC8CPgtPimF4Eb3H3boEU5QKmSCILUFXWWbt5LOGTMmlBG6ABt9K0dXeSFs4bsh226i3RFqdpaT1N7hOK8bApzs3GHys17Wby+jsUb6qhr7iAvnMW5M0fzsdnjOGt6uTqpM9CNv6piycY9vHTz3w36sftLBAP9+nE/sSklLo4/viy+7QOHH570Fsoy5k0ZPuDy+Tn6wEhl2aEsKibv+/ecObaEy08+gmjUeW3LXh6t2sETK6t5fEU1BTkhzp5RzvnHjuGco0f1W5uUoaM2wWsVdxtoIih39/t7PP6ZmX05iIBEMk1WllExeTgVk4fzbx89hpfX1/HUGzv506pdPLFyJ3nhLOafOY0FZ02lIEdNR0NZXVM7Y0ryEn7egQ5nqDOzy8wsFL9dBtQFGZhIJsoOZXHm9HL+48L3seSfz+WRfziVc2eO5s7n1vJ33/sLv1+2negABiZIekrG9BIw8ERwNfD3wE6gGrgIuDKgmESEWE1h7hHD+O/PnMBvFpxCeXEuX/5VFZf8ZDHb9rYkOzwZZO5OXXN7wi8mgwEmAnff7O4fc/dydx/l7p8AEj50VCRTnTh5OI9edxq3fvJ9vLGjgQtuf5HfL9tOuo36k/1raItd5Z7o6SXg8Jaq7Hd6CREZXFlZxqdOnMSTN5zBjDHFfPlXVXzpl1Xsjc9PI+ktGWsVdzucRKCxiiJJMHF4Ab/6/CncdP4MnlxZzd99/3l+XblVtYM0t7sxPROB3nUiSRLKMq4750ge+9LpTCsv4p8WreBT/7OYt3Y1Jjs0OUTVb7cCMK4sxUYNmVmjmTX0cWsExiUoRhHZj6PHlPDrz5/CrZ98H2/tbuRDd7zI955+k7bOA0+4J6llR31sNoFxZfkJP3e/icDdi929pI9bsbtrQLNICujuO3juxrP42Oxx3PV/6/jQHS+yeINGeKeT7fWtjCjMScoV5ZoWUWSIGFGUyw8+NYeHrplHZzTKpxcu5lt/XEWka/8zz0rq2FHfmpTaACgRiAw5ZxxVzjNfPosrT53MfS9t5Mr7/0Z9i0YWpbpYIkh8/wAoEYgMSfk5Ib75sWP5r4tm8erGPXz8v19irTqSU5a7s32vagQiEoC/r5jIL+afRHN7Fxf++GVeXl+b7JCkDw1tEZo7uhivRCAiQZh7xHD+8MXTGFeWx5X3/41nV+1KdkjSy4767qGjSgQiEpBxZfn8av4pHD2mmM//71Ierdqe7JCkByUCEUmIYYU5PPy5k6g4Yhhf/lUVD72yKdkhSdy7iWAIdhab2SYzW2lmVWa2z7JiZlZqZn80s+Vm9oaZXRVkPCKZrjgvzANXz+OcGaP4xqNv8I3fv06nhpcm3fb6NnJCWYwsTM6a3omoEZzj7nP2s0TadcAqd58NnA1838wSP/WeSAbJC4dYePlcrj1jCg8t3syl9y6hNj7hmSTHjvpWxpblkXWAZWmDkuymIQeKLbbYbhGwB4gkNySRoS87lMXXP3wMt39qDsu31vOxH/2VN3dqeGmy7KhvZVxpcvoHIPhE4MAzZrbUzOb3sf8uYCawA1gJ3ODu+9RTzWy+mVWaWWVNTU2wEYtkkE8cP55FC04lEnWuuG8J2+Nt1ZJYybyqGIJPBKe7+wnABcB1ZnZmr/3nA1XEJrCbA9xlZiW9D+LuC929wt0rysvLAw5ZJLO8b0IpD14zj5aOLj5736u6CjnBIl1Rdja0MT5JHcUQcCJw9+3xn7uB3wHzehW5Cvitx6wDNgJHBxmTiOzr6DElLLy8gi11LXzugUrNXppAuxrbiXryho5CgInAzArNrLj7PnAe8HqvYluAc+NlRgMzgA1BxSQi+3fKtBH88FNzWLplLzf8chldUS05kgjJvoYAgq0RjAb+ambLgVeBx939KTNbYGYL4mW+DZxqZiuB54CvubuugRdJkg/PGsu/fuQYnn5jF//xxOpkh5MRUiERBLamgLtvAGb3sf2eHvd3EKspiEiKuOq0KWyua+Gnf93I5JGFXH7yEckOaUjbnuSLySDARCAi6esbHzmGLXta+OYf3mDS8ALOmq5BGkHZUd/KsIIwBTnJ+zhO9nUEIpKCQlnGnZccz/TRxVz38Gus2dmQ7JCGrB31bUltFgIlAhHZj6LcbO67soLC3BDX/KxSVx8HJNnXEIASgYj0Y2xpPj+5ooK65nYWPLSU9oiGlQ627fWtSVuHoJsSgYj0a9aEMr538WwqN+/ln3+7EncNKx0sDW2dNLZFktpRDOosFpEB+Miscazb3cTtz65l+uhiFpw1LdkhDQnV9W1AcoeOgmoEIjJAN5x7FB+eNZZbn1rD/725O9nhDAmpcA0BKBGIyACZGd+7aDZHjynhK7+qeudDTA5d9zUE6iMQkbSRnxPivz9zPJ2RKF/8+Wta1OYw7ahvJRwyyouSsyBNNyUCETkoU8uL+O4nZ/Halnpue/rNZIeT1nbUtzKmNHkL0nRTIhCRg/bR2eO47ORJLHxhA8+u2pXscNLWjvq2pC5I002JQEQOyf/78DEcN76EG39dxbrdTckOJy1tqmtmwrCCZIehRCAihyYvHOLuS+eSk53F1T/7G3uataDNwahramd3YzszxxYnOxQlAhE5dBOHF7Dwigp2NbQx/0EtaHMw1sTXiJ45dp9FGRNOiUBEDssJk4bxg7+fQ+XmvXztkRW68niAVlfHJvI7eoxqBCIyBHx41lhuOn8Gj1bt4PZn1yY7nLSwurqRUcW5jEjy0FHQFBMiMki+cPY0NtY2c8dza5k2qoiPzR6X7JBS2urqBo5OgWYhUI1ARAaJmXHLhccxb/Jw/vE3y1m2ZW+yQ0pZnV1R1u1uSomOYlAiEJFBlJsd4p7L5zK6JJdrH1yqaSj2Y0NNMx1dUWaOUY1ARIag4YU53PfZE2nv7OKaBypp7dBIot66V3xLhRFDoEQgIgE4anQxd37meFZXN3DrU2uSHU7KWVXdQE4oi6nlhckOBQg4EZjZJjNbaWZVZla5nzJnx/e/YWZ/CTIeEUmcc2aM4spTJ/Pnx19h52VXQ0kJZGXFfn7hC7B+fbJDTJo11Y0cOaqIcCg1vosnYtTQOe5e29cOMysDfgx80N23mNmoBMQjIglyMxv52v3Xk93VCdF4E1FjI9x7LzzwACxaBBdckNwgk2B1dQOnHzUy2WG8I9np6DPAb919C4C7a7ULkaFi/XryLvkU+Z1thKO9+gk6O6GlBS66KONqBt1TSxyTIv0DEHwicOAZM1tqZvP72D8dGGZmz8fLXBFwPCKSKN//fuwDvz+dnfDDHyYmnhTRPbXE0SkyYgiCTwSnu/sJwAXAdWZ2Zq/92cBc4MPA+cA3zGx674OY2XwzqzSzypqamoBDFpFB8b//O7BE8NBDiYknRXRPLZEq1xBAwInA3bfHf+4GfgfM61VkG/C0uzfH+xFeAGb3cZyF7l7h7hXl5eVBhiwig6VpgFNTD7TcELG6upHyFJlaoltgicDMCs2suPs+cB7weq9ijwKnm1m2mRUAJwGrg4pJRBKoqGhwyw0Ra3Y2pMz1A92CrBGMBv5qZsuBV4HH3f0pM1tgZgsA3H018BSwIl7mXnfvnSxEJB1ddhmEw/2XCYfh8ssTE08K6OyKsnZXEzNTYMbRngIbPuruG+i7meeeXo9vA24LKg4RSZKvfjU2RLSffgIPh7GvfCWBQSXXxtr41BIZVCMQkUw2bVrsOoGCgn1qBtHsbFqyc/ndv9weK5ch3lmDIIU6ikGJQESCdMEFsGIFzJ//niuLbf587vzBIm5sHMejVduTHWXCLNtST144i2nlqdUvovUIRCRY06bBXXfFbnEG3BiJ8tq9S7hp0QomDi/ghEnDkhdjgry8vpYTJw9PmakluqVWNCKSMXKys7jn8rmMKclj/oOVbNvbkuyQArW7sY23djVx2pGpM7VENyUCEUma4YU53HdlBe2RKJ8b4lNWv7K+DoBTp41IciT7UiIQkaQ6clQxP7rkeN7c1ci3HluV7HAC8/K6Okrysjl2XGmyQ9mHEoGIJN3ZM0bx+TOn8YtXt/DkyupkhxOIlzfUcvLUEYSyLNmh7EOJQERSwlfPm87siWV87ZEVbB9iS1xu3dPC1j2tKdksBEoEIpIiwqEs7vz0HKIOX/llFZGuaLJDGjQvr48tyZKKHcWgRCAiKeSIEYXccuFxvLppD3c+tzbZ4Qyal9bVUV6cy5GjUuv6gW5KBCKSUj4+ZzwXz53AnX9ex9Nv7Ex2OIfN3Xl5fR2nThuBWer1D4ASgYikoG9/4jhmTyzjxl9V8dauxmSHc1jW7W6itqk9ZfsHQIlARFJQXjjEwsvnUpCbzbUPVlLf0pHskA7ZS+ti/QOnTkvN/gFQIhCRFDW6JI97LptLdX0b1/9iWdp2Hr+8vo6Jw/OZOLwg2aHslxKBiKSsuUcM4zufOI4X19by/T+9lexwDlpX1Fm8oY5Tp6ZubQCUCEQkxf39iRO5ZN5E7n5+Pc+t3pXscA5K5aY9NLRFOGO6EoGIyGH5t48eyzFjS7jx18vZuid9Jqd78vWd5GZncfaMUckOpV9KBCKS8vLCIe6+7ASiUeeLP3+Njkjq9xdEo85Tr+/krOnlFOWm9oz/SgQikhaOGFHIbRfPYvm2t7nl8dSfnG7Z1np2NrRxwfvGJDuUA1IiEJG08cHjxnLtGVN44JXN/PSvG5MdTr+eer2acMg4d+boZIdyQKldXxER6eXmC2aydU8r335sFeXFuXxs9rhkh7QPd+eJlTs546hySvLCB35CkqlGICJpJZRl3P7pOcybMpyv/rqKl+MXbKWSldvfZnt9Kx88LvWbhSDgRGBmm8xspZlVmVllP+VONLOImV0UZDwiMjTkhUP85IoKpo4sYv5DS3l9+9vJDuk9nnx9J9lZxnnHpH6zECSmRnCOu89x94q+dppZCLgVeCYBsYjIEFGaH+ZnV59ISV42V9z3KmtTZE4id+fJldWcMm0EZQU5yQ5nQFKhaeh64BFgd7IDEZH0MrY0n4evPZlQlnHpvUvYVNuc7JBYs7ORTXUtXHDc2GSHMmBBJwIHnjGzpWY2v/dOMxsPXAjc3d9BzGy+mVWaWWVNTU1AoYpIOpoyspCHP3cSnV1RLr13SdJXN3tyZTVZBucdmx7NQhB8Ijjd3U8ALgCuM7Mze+2/Hfiau/d7dYi7L3T3CnevKC8vDypWEUlT00cX89A1J9HQ1smlP1nMjiQlg+b2CD9/dQunHTmSkUW5SYnhUASaCNx9e/znbuB3wLxeRSqAX5rZJuAi4Mdm9okgYxKRoem48aU8cPU86po6uOjul1lf05TwGH7y4gZqmzq48QPTE37uwxFYIjCzQjMr7r4PnAe83rOMu09x98nuPhlYBHzB3X8fVEwiMrSdMGkYv/z8yXR0Rbn4nlcSOpqoprGdn7ywgQuOG8Pxk4Yl7LyDIcgawWjgr2a2HHgVeNzdnzKzBWa2IMDzikgGO3ZcKb9ZcCr54RCfXriYV9bXJeS8d/15LW2RKDedPyMh5xtM5u7JjuGgVMPSut8AAAo1SURBVFRUeGXlfi9JEBEBYOfbbVxx3xI21bZw28Wz+Pic8YGda1NtM+//wV/41IkTueXC9wV2nsNhZkv3N4w/FYaPiogMujGlefzm86dy/KQybvhlFf/9f+sI6ovv9555k3Aoixvef1Qgxw+aEoGIDFmlBWEevGYen5gzjtuefpObH1lJ5yAvefnyuloeW1HNtWdMYVRx3qAeO1E06ZyIDGm52SF++Kk5TBxewI/+vI61uxu549PHD8oawks37+XaBys5clQR1545dRCiTQ7VCERkyDMzvnreDH50yfGs3dXEh+54kT8u33FYx1y57W2uvP9Vyotz+fnnTqI4DWYZ3R8lAhHJGB+dPY4nbjiDI0cXcf0vlnHjr6vYXHfw01Ksrm7g8vuWUJIX5uFrT2ZUSXo2CXXTqCERyTidXVHueHYt9/xlPV3ufGDmaK45fQrzpgzHzPb7vB31rdz/0kZ+vmQLxXlhfv35U5g04vCbmBKhv1FDSgQikrF2NbTx4CubeHjJFupbOplWXsj5x47h/GPHMGtCKZGos7muhXW7m3j6jZ38cfkOHPjIrLF89QMz0iYJgBKBiEi/Wju6+H3Vdv64fAdLNu6hK+oMKwjT2BYhEo19RhbkhPj0iZO4+vTJTBiWPgmgW3+JQKOGRCTj5eeEuGTeJC6ZN4m9zR08t2Y3izfUMao4lyNHFTGtvIijRhdRkDM0PzKH5m8lInKIhhXmcNHcCVw0d0KyQ0kYjRoSEclwSgQiIhlOiUBEJMMpEYiIZDglAhGRDKdEICKS4ZQIREQynBKBiEiGS7spJsysBqgHeq9KXXqAbQe63/1zJFB7CKH1df6B7O+9vb/HvWPtue1Q4k5kzD3vJ+O11vtD74/+9qfj++NgYgY4yt1L+zy6u6fdDVh4sNsOdL/Hz8rBimkg+3tv7+9x71gPN+5Expzs11rvD70/htr742BiPtA50rVp6I+HsO1A9/t6/uHGNJD9vbf397ivWA8n7kTG3PN+Ml5rvT8Ont4fA7+f6jH3e460axoKmplV+n5m6Etl6Ri3Yk6cdIxbMSdOutYIgrQw2QEconSMWzEnTjrGrZgTRDUCEZEMpxqBiEiGUyIQEclwQzoRmNl9ZrbbzF4/hOfONbOVZrbOzO60Hitam9n1ZrbGzN4ws/8a3KiDidvMvmlm282sKn77UKrH3GP/V83MzWzk4EUc2Ov8bTNbEX+NnzGzcWkQ823x9/MKM/udmZUNZswBxn1x/H8wamaD1kF7OLHu53ifNbO18dtne2zv932fUIcy5jVdbsCZwAnA64fw3FeBkwEDngQuiG8/B3gWyI0/HpUmcX8T+Md0eq3j+yYCTwObgZGpHjNQ0qPMl4B70iDm84Ds+P1bgVvT4f0BzARmAM8DFcmONR7H5F7bhgMb4j+Hxe8P6+/3SsZtSNcI3P0FYE/PbWY2zcyeMrOlZvaimR3d+3lmNpbYP/Rij/3FHgQ+Ed/9D8B33b09fo7daRJ3oAKM+YfAPwGDPqohiJjdvaFH0cLBjjugmJ9x90i86GJg0NdoDCju1e7+ZqrEuh/nA39y9z3uvhf4E/DBZP6v9mVIJ4L9WAhc7+5zgX8EftxHmfHAth6Pt8W3AUwHzjCzJWb2FzM7MdBo33W4cQN8MV79v8/MhgUX6jsOK2Yz+ziw3d2XBx1oD4f9OpvZLWa2FbgU+NcAY+02GO+NblcT+3aaCIMZd9AGEmtfxgNbezzujj9Vfi8gwxavN7Mi4FTgNz2a43IP8jDZxKp5JwMnAr82s6nxrB6IQYr7buDbxL6hfhv4PrF/+kAcbsxmVgD8C7Fmi4QYpNcZd/868HUz+2fgi8C/DVqQvQxWzPFjfR2IAA8PTnT9nmvQ4g5af7Ga2VXADfFtRwJPmFkHsNHdL0x0rIcqoxIBsRpQvbvP6bnRzELA0vjDPxD70OxZPZ4AbI/f3wb8Nv7B/6qZRYlNNFWTynG7+64ez/sJ8FiA8cLhxzwNmAIsj//zTQBeM7N57r4zRWPu7WHgCQJMBAxSzGZ2JfAR4Nwgv9T0MNivdZD6jBXA3e8H7gcws+eBK919U48i24GzezyeQKwvYTvJ/73elazOiUTdgMn06PQBXgYujt83YPZ+nte7I+dD8e0LgG/F708nVu2zNIh7bI8yXwF+meox9yqziUHuLA7odT6qR5nrgUVpEPMHgVVA+WDHmoj3B4PcWXyosbL/zuKNxDqKh8XvDx/o+z5Rt6ScNGG/HPwCqAY6iX2Tv4bYt8yngOXxN/+/7ue5FcDrwHrgLt69CjsH+N/4vteAv0uTuB8CVgIriH3TGpvqMfcqs4nBHzUUxOv8SHz7CmKTfI1Pg5jXEftCUxW/DepIpwDjvjB+rHZgF/B0MmOlj0QQ3351/DVeB1x1MO/7RN00xYSISIbLxFFDIiLSgxKBiEiGUyIQEclwSgQiIhlOiUBEJMMpEciQYGZNCT7fvWZ2zCAdq8tis5W+bmZ/PNDsn2ZWZmZfGIxzi4BWKJMhwsya3L1oEI+X7e9OxBaonrGb2QPAW+5+Sz/lJwOPuftxiYhPhj7VCGTIMrNyM3vEzP4Wv50W3z7PzF4xs2Vm9rKZzYhvv9LM/mBmfwaeM7Ozzex5M1tksfn6H+6eMz6+vSJ+vyk+0dxyM1tsZqPj26fFH680s+8MsNbyCu9OuldkZs+Z2WvxY3w8Xua7wLR4LeK2eNmb4r/jCjP790F8GSUDKBHIUHYH8EN3PxH4JHBvfPsa4Ax3P57Y7KD/0eM5JwAXuftZ8cfHA18GjgGmAqf1cZ5CYLG7zwZeAK7tcf473P19vHemyT7F59k5l9iV3wBtwIXufgKxdTC+H09ENwPr3X2Ou99kZucBRwHzgDnAXDM780DnE+mWaZPOSWZ5P3BMjxkjS+IzSZYCD5jZUcRmYw33eM6f3L3nXPSvuvs2ADOrIjYHzV97naeDdyfxWwp8IH7/FN6dY/7nwPf2E2d+/NjjgdXE5qyH2Bw0/xH/UI/G94/u4/nnxW/L4o+LiCWGF/ZzPpH3UCKQoSwLONnd23puNLO7gP9z9wvj7e3P99jd3OsY7T3ud9H3/0ynv9vZtr8y/Wl19znxqbefBq4D7iS2nkE5MNfdO81sE5DXx/MN+E93/5+DPK8IoKYhGdqeITYDKABm1j2NcCnvTvl7ZYDnX0ysSQrg0wcq7O4txJa3/KqZZROLc3c8CZwDHBEv2ggU93jq08DV8doOZjbezEYN0u8gGUCJQIaKAjPb1uN2I7EP1Yp4B+oqYlOIA/wX8J9mtoxga8VfBm40sxXEFi15+0BPcPdlxGYuvYTYegYVZrYSuIJY3wbuXge8FB9uepu7P0Os6emVeNlFvDdRiPRLw0dFAhJv6ml1dzezTwOXuPvHD/Q8kURTH4FIcOYCd8VH+tQT4NKgIodDNQIRkQynPgIRkQynRCAikuGUCEREMpwSgYhIhlMiEBHJcP8fP+E2RhbAgvMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.lr_find();learn.recorder.plot(suggestion = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='18' class='' max='100', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      18.00% [18/100 04:49<21:58]\n",
       "    </div>\n",
       "    \n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>5.173929</td>\n",
       "      <td>4.227953</td>\n",
       "      <td>0.331885</td>\n",
       "      <td>00:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>4.568038</td>\n",
       "      <td>3.730512</td>\n",
       "      <td>0.379928</td>\n",
       "      <td>00:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>4.152972</td>\n",
       "      <td>3.506199</td>\n",
       "      <td>0.402661</td>\n",
       "      <td>00:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>3.851606</td>\n",
       "      <td>3.353625</td>\n",
       "      <td>0.422802</td>\n",
       "      <td>00:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>3.630903</td>\n",
       "      <td>3.251747</td>\n",
       "      <td>0.435113</td>\n",
       "      <td>00:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>3.427743</td>\n",
       "      <td>3.168539</td>\n",
       "      <td>0.445896</td>\n",
       "      <td>00:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>3.259858</td>\n",
       "      <td>3.120778</td>\n",
       "      <td>0.452335</td>\n",
       "      <td>00:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>3.109029</td>\n",
       "      <td>3.089926</td>\n",
       "      <td>0.458499</td>\n",
       "      <td>00:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>3.013727</td>\n",
       "      <td>3.090643</td>\n",
       "      <td>0.459444</td>\n",
       "      <td>00:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>2.914305</td>\n",
       "      <td>3.093919</td>\n",
       "      <td>0.461229</td>\n",
       "      <td>00:09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>2.834301</td>\n",
       "      <td>3.104919</td>\n",
       "      <td>0.462706</td>\n",
       "      <td>00:13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>2.784172</td>\n",
       "      <td>3.133152</td>\n",
       "      <td>0.462139</td>\n",
       "      <td>00:22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>2.758499</td>\n",
       "      <td>3.161335</td>\n",
       "      <td>0.462139</td>\n",
       "      <td>00:22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>2.747886</td>\n",
       "      <td>3.182941</td>\n",
       "      <td>0.462620</td>\n",
       "      <td>00:22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>2.729133</td>\n",
       "      <td>3.223763</td>\n",
       "      <td>0.460337</td>\n",
       "      <td>00:27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>2.744855</td>\n",
       "      <td>3.238824</td>\n",
       "      <td>0.462775</td>\n",
       "      <td>00:35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>16</td>\n",
       "      <td>2.747836</td>\n",
       "      <td>3.268671</td>\n",
       "      <td>0.458894</td>\n",
       "      <td>00:40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>17</td>\n",
       "      <td>2.765789</td>\n",
       "      <td>3.294989</td>\n",
       "      <td>0.458568</td>\n",
       "      <td>00:41</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>\n",
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:04<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 18: early stopping\n"
     ]
    }
   ],
   "source": [
    "learn.fit_one_cycle(100, 5e-2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='100', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      13.00% [13/100 12:15<1:21:59]\n",
       "    </div>\n",
       "    \n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>2.367440</td>\n",
       "      <td>3.147998</td>\n",
       "      <td>0.478348</td>\n",
       "      <td>00:33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>2.309863</td>\n",
       "      <td>3.129632</td>\n",
       "      <td>0.483620</td>\n",
       "      <td>00:48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>2.220500</td>\n",
       "      <td>3.121608</td>\n",
       "      <td>0.485577</td>\n",
       "      <td>00:56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>2.127625</td>\n",
       "      <td>3.127086</td>\n",
       "      <td>0.488616</td>\n",
       "      <td>00:55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>2.071890</td>\n",
       "      <td>3.131075</td>\n",
       "      <td>0.490419</td>\n",
       "      <td>00:56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>1.989892</td>\n",
       "      <td>3.144920</td>\n",
       "      <td>0.491295</td>\n",
       "      <td>00:57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>1.906401</td>\n",
       "      <td>3.168906</td>\n",
       "      <td>0.492016</td>\n",
       "      <td>00:57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>1.827848</td>\n",
       "      <td>3.190956</td>\n",
       "      <td>0.494008</td>\n",
       "      <td>00:57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>1.749057</td>\n",
       "      <td>3.208365</td>\n",
       "      <td>0.493561</td>\n",
       "      <td>00:57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>1.675212</td>\n",
       "      <td>3.279721</td>\n",
       "      <td>0.492994</td>\n",
       "      <td>00:57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>1.611470</td>\n",
       "      <td>3.297421</td>\n",
       "      <td>0.493647</td>\n",
       "      <td>00:57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>1.551534</td>\n",
       "      <td>3.321082</td>\n",
       "      <td>0.489766</td>\n",
       "      <td>00:57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>1.496373</td>\n",
       "      <td>3.382489</td>\n",
       "      <td>0.489234</td>\n",
       "      <td>00:57</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>\n",
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:04<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Better model found at epoch 0 with accuracy value: 0.4783481955528259.\n",
      "Better model found at epoch 0 with accuracy value: 0.4783481955528259.\n",
      "Better model found at epoch 0 with accuracy value: 0.4783481955528259.\n",
      "Better model found at epoch 1 with accuracy value: 0.4836195111274719.\n",
      "Better model found at epoch 1 with accuracy value: 0.4836195111274719.\n",
      "Better model found at epoch 1 with accuracy value: 0.4836195111274719.\n",
      "Better model found at epoch 2 with accuracy value: 0.48557692766189575.\n",
      "Better model found at epoch 2 with accuracy value: 0.48557692766189575.\n",
      "Better model found at epoch 2 with accuracy value: 0.48557692766189575.\n",
      "Better model found at epoch 3 with accuracy value: 0.48861610889434814.\n",
      "Better model found at epoch 3 with accuracy value: 0.48861610889434814.\n",
      "Better model found at epoch 3 with accuracy value: 0.48861610889434814.\n",
      "Better model found at epoch 4 with accuracy value: 0.49041903018951416.\n",
      "Better model found at epoch 4 with accuracy value: 0.49041903018951416.\n",
      "Better model found at epoch 4 with accuracy value: 0.49041903018951416.\n",
      "Better model found at epoch 5 with accuracy value: 0.49129465222358704.\n",
      "Better model found at epoch 5 with accuracy value: 0.49129465222358704.\n",
      "Better model found at epoch 5 with accuracy value: 0.49129465222358704.\n",
      "Better model found at epoch 6 with accuracy value: 0.4920158088207245.\n",
      "Better model found at epoch 6 with accuracy value: 0.4920158088207245.\n",
      "Better model found at epoch 6 with accuracy value: 0.4920158088207245.\n",
      "Better model found at epoch 7 with accuracy value: 0.49400755763053894.\n",
      "Better model found at epoch 7 with accuracy value: 0.49400755763053894.\n",
      "Better model found at epoch 7 with accuracy value: 0.49400755763053894.\n",
      "Epoch 13: early stopping\n"
     ]
    }
   ],
   "source": [
    "learn.callbacks.append(SaveModelCallback(learn, monitor='accuracy'))\n",
    "learn.unfreeze()\n",
    "learn.fit_one_cycle(100, 1e-2, wd = 0.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LanguageLearner(data=TextLMDataBunch;\n",
       "\n",
       "Train: LabelList (8700 items)\n",
       "x: LMTextList\n",
       "xxbos # xxmaj thorium xxmaj radioactive xxmaj weapons . xxmaj xxunk murders and environmental devastation : - xxup video http : / / t.co / xxunk,xxbos @_minimehh @cjoyner i must be overlooking the burning buildings ? # blacklivesmatter,xxbos xxmaj economic xxmaj collapse xxmaj xxunk : xxmaj specific actions and xxunk to securing lasting wealth from the financial blowout . http : / / t.co / xxunk,xxbos xxunk we will never know what would have happened but the govt seemed to think that their xxunk xxunk the deaths of innocent xxunk,xxbos xxmaj first xxmaj responders xxmaj xxunk for xxmaj national xxmaj summit and xxmaj xxunk on xxup xxunk xxmaj technology http : / / t.co / xxunk # xxunk # xxunk\n",
       "y: LMLabelList\n",
       ",,,,\n",
       "Path: .;\n",
       "\n",
       "Valid: LabelList (2176 items)\n",
       "x: LMTextList\n",
       "xxbos xxmaj xxunk xxmaj xxunk xxmaj health xxmaj care xxmaj reviews xxmaj with xxmaj xxunk xxmaj journalism : xxmaj sick and injured xxunk at a local xxup er are t ... http : / / t.co / xxunk,xxbos xxup lol xxmaj xxunk in the midst of xxunk meltdown reaching for xxmaj double xxmaj xxunk . # xxmaj mets,xxbos xxmaj choking xxmaj hazard xxmaj prompts xxmaj recall xxmaj of xxmaj xxunk xxmaj cheese xxmaj singles http : / / t.co / xxunk,xxbos xxmaj what a night ! xxmaj xxunk go on vacation to end our xxmaj dream xxmaj job party where fire truck xxunk the pool with a xxunk ! http : / / t.co / xxunk,xxbos who said this ? xxmaj xxunk xxmaj sam or xxmaj xxunk ? xxunk xxunk vote for a flattened out xxunk skin a - huh huh xxup û _ i always xxunk\n",
       "y: LMLabelList\n",
       ",,,,\n",
       "Path: .;\n",
       "\n",
       "Test: None, model=SequentialRNN(\n",
       "  (0): AWD_LSTM(\n",
       "    (encoder): Embedding(7520, 400, padding_idx=1)\n",
       "    (encoder_dp): EmbeddingDropout(\n",
       "      (emb): Embedding(7520, 400, padding_idx=1)\n",
       "    )\n",
       "    (rnns): ModuleList(\n",
       "      (0): WeightDropout(\n",
       "        (module): LSTM(400, 1152, batch_first=True)\n",
       "      )\n",
       "      (1): WeightDropout(\n",
       "        (module): LSTM(1152, 1152, batch_first=True)\n",
       "      )\n",
       "      (2): WeightDropout(\n",
       "        (module): LSTM(1152, 400, batch_first=True)\n",
       "      )\n",
       "    )\n",
       "    (input_dp): RNNDropout()\n",
       "    (hidden_dps): ModuleList(\n",
       "      (0): RNNDropout()\n",
       "      (1): RNNDropout()\n",
       "      (2): RNNDropout()\n",
       "    )\n",
       "  )\n",
       "  (1): LinearDecoder(\n",
       "    (decoder): Linear(in_features=400, out_features=7520, bias=True)\n",
       "    (output_dp): RNNDropout()\n",
       "  )\n",
       "), opt_func=functools.partial(<class 'torch.optim.adam.Adam'>, betas=(0.9, 0.99)), loss_func=FlattenedLoss of CrossEntropyLoss(), metrics=[<function accuracy at 0x7f58083928c8>], true_wd=True, bn_wd=True, wd=0.01, train_bn=True, path=PosixPath('.'), model_dir='models', callback_fns=[functools.partial(<class 'fastai.basic_train.Recorder'>, add_time=True, silent=False)], callbacks=[RNNTrainer\n",
       "learn: LanguageLearner(data=TextLMDataBunch;\n",
       "\n",
       "Train: LabelList (8700 items)\n",
       "x: LMTextList\n",
       "xxbos # xxmaj thorium xxmaj radioactive xxmaj weapons . xxmaj xxunk murders and environmental devastation : - xxup video http : / / t.co / xxunk,xxbos @_minimehh @cjoyner i must be overlooking the burning buildings ? # blacklivesmatter,xxbos xxmaj economic xxmaj collapse xxmaj xxunk : xxmaj specific actions and xxunk to securing lasting wealth from the financial blowout . http : / / t.co / xxunk,xxbos xxunk we will never know what would have happened but the govt seemed to think that their xxunk xxunk the deaths of innocent xxunk,xxbos xxmaj first xxmaj responders xxmaj xxunk for xxmaj national xxmaj summit and xxmaj xxunk on xxup xxunk xxmaj technology http : / / t.co / xxunk # xxunk # xxunk\n",
       "y: LMLabelList\n",
       ",,,,\n",
       "Path: .;\n",
       "\n",
       "Valid: LabelList (2176 items)\n",
       "x: LMTextList\n",
       "xxbos xxmaj xxunk xxmaj xxunk xxmaj health xxmaj care xxmaj reviews xxmaj with xxmaj xxunk xxmaj journalism : xxmaj sick and injured xxunk at a local xxup er are t ... http : / / t.co / xxunk,xxbos xxup lol xxmaj xxunk in the midst of xxunk meltdown reaching for xxmaj double xxmaj xxunk . # xxmaj mets,xxbos xxmaj choking xxmaj hazard xxmaj prompts xxmaj recall xxmaj of xxmaj xxunk xxmaj cheese xxmaj singles http : / / t.co / xxunk,xxbos xxmaj what a night ! xxmaj xxunk go on vacation to end our xxmaj dream xxmaj job party where fire truck xxunk the pool with a xxunk ! http : / / t.co / xxunk,xxbos who said this ? xxmaj xxunk xxmaj sam or xxmaj xxunk ? xxunk xxunk vote for a flattened out xxunk skin a - huh huh xxup û _ i always xxunk\n",
       "y: LMLabelList\n",
       ",,,,\n",
       "Path: .;\n",
       "\n",
       "Test: None, model=SequentialRNN(\n",
       "  (0): AWD_LSTM(\n",
       "    (encoder): Embedding(7520, 400, padding_idx=1)\n",
       "    (encoder_dp): EmbeddingDropout(\n",
       "      (emb): Embedding(7520, 400, padding_idx=1)\n",
       "    )\n",
       "    (rnns): ModuleList(\n",
       "      (0): WeightDropout(\n",
       "        (module): LSTM(400, 1152, batch_first=True)\n",
       "      )\n",
       "      (1): WeightDropout(\n",
       "        (module): LSTM(1152, 1152, batch_first=True)\n",
       "      )\n",
       "      (2): WeightDropout(\n",
       "        (module): LSTM(1152, 400, batch_first=True)\n",
       "      )\n",
       "    )\n",
       "    (input_dp): RNNDropout()\n",
       "    (hidden_dps): ModuleList(\n",
       "      (0): RNNDropout()\n",
       "      (1): RNNDropout()\n",
       "      (2): RNNDropout()\n",
       "    )\n",
       "  )\n",
       "  (1): LinearDecoder(\n",
       "    (decoder): Linear(in_features=400, out_features=7520, bias=True)\n",
       "    (output_dp): RNNDropout()\n",
       "  )\n",
       "), opt_func=functools.partial(<class 'torch.optim.adam.Adam'>, betas=(0.9, 0.99)), loss_func=FlattenedLoss of CrossEntropyLoss(), metrics=[<function accuracy at 0x7f58083928c8>], true_wd=True, bn_wd=True, wd=0.01, train_bn=True, path=PosixPath('.'), model_dir='models', callback_fns=[functools.partial(<class 'fastai.basic_train.Recorder'>, add_time=True, silent=False)], callbacks=[...], layer_groups=[Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(400, 1152, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1152, 1152, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1152, 400, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): Embedding(7520, 400, padding_idx=1)\n",
       "  (1): EmbeddingDropout(\n",
       "    (emb): Embedding(7520, 400, padding_idx=1)\n",
       "  )\n",
       "  (2): LinearDecoder(\n",
       "    (decoder): Linear(in_features=400, out_features=7520, bias=True)\n",
       "    (output_dp): RNNDropout()\n",
       "  )\n",
       ")], add_time=True, silent=False)\n",
       "alpha: 2.0\n",
       "beta: 1.0, EarlyStoppingCallback\n",
       "learn: LanguageLearner(data=TextLMDataBunch;\n",
       "\n",
       "Train: LabelList (8700 items)\n",
       "x: LMTextList\n",
       "xxbos # xxmaj thorium xxmaj radioactive xxmaj weapons . xxmaj xxunk murders and environmental devastation : - xxup video http : / / t.co / xxunk,xxbos @_minimehh @cjoyner i must be overlooking the burning buildings ? # blacklivesmatter,xxbos xxmaj economic xxmaj collapse xxmaj xxunk : xxmaj specific actions and xxunk to securing lasting wealth from the financial blowout . http : / / t.co / xxunk,xxbos xxunk we will never know what would have happened but the govt seemed to think that their xxunk xxunk the deaths of innocent xxunk,xxbos xxmaj first xxmaj responders xxmaj xxunk for xxmaj national xxmaj summit and xxmaj xxunk on xxup xxunk xxmaj technology http : / / t.co / xxunk # xxunk # xxunk\n",
       "y: LMLabelList\n",
       ",,,,\n",
       "Path: .;\n",
       "\n",
       "Valid: LabelList (2176 items)\n",
       "x: LMTextList\n",
       "xxbos xxmaj xxunk xxmaj xxunk xxmaj health xxmaj care xxmaj reviews xxmaj with xxmaj xxunk xxmaj journalism : xxmaj sick and injured xxunk at a local xxup er are t ... http : / / t.co / xxunk,xxbos xxup lol xxmaj xxunk in the midst of xxunk meltdown reaching for xxmaj double xxmaj xxunk . # xxmaj mets,xxbos xxmaj choking xxmaj hazard xxmaj prompts xxmaj recall xxmaj of xxmaj xxunk xxmaj cheese xxmaj singles http : / / t.co / xxunk,xxbos xxmaj what a night ! xxmaj xxunk go on vacation to end our xxmaj dream xxmaj job party where fire truck xxunk the pool with a xxunk ! http : / / t.co / xxunk,xxbos who said this ? xxmaj xxunk xxmaj sam or xxmaj xxunk ? xxunk xxunk vote for a flattened out xxunk skin a - huh huh xxup û _ i always xxunk\n",
       "y: LMLabelList\n",
       ",,,,\n",
       "Path: .;\n",
       "\n",
       "Test: None, model=SequentialRNN(\n",
       "  (0): AWD_LSTM(\n",
       "    (encoder): Embedding(7520, 400, padding_idx=1)\n",
       "    (encoder_dp): EmbeddingDropout(\n",
       "      (emb): Embedding(7520, 400, padding_idx=1)\n",
       "    )\n",
       "    (rnns): ModuleList(\n",
       "      (0): WeightDropout(\n",
       "        (module): LSTM(400, 1152, batch_first=True)\n",
       "      )\n",
       "      (1): WeightDropout(\n",
       "        (module): LSTM(1152, 1152, batch_first=True)\n",
       "      )\n",
       "      (2): WeightDropout(\n",
       "        (module): LSTM(1152, 400, batch_first=True)\n",
       "      )\n",
       "    )\n",
       "    (input_dp): RNNDropout()\n",
       "    (hidden_dps): ModuleList(\n",
       "      (0): RNNDropout()\n",
       "      (1): RNNDropout()\n",
       "      (2): RNNDropout()\n",
       "    )\n",
       "  )\n",
       "  (1): LinearDecoder(\n",
       "    (decoder): Linear(in_features=400, out_features=7520, bias=True)\n",
       "    (output_dp): RNNDropout()\n",
       "  )\n",
       "), opt_func=functools.partial(<class 'torch.optim.adam.Adam'>, betas=(0.9, 0.99)), loss_func=FlattenedLoss of CrossEntropyLoss(), metrics=[<function accuracy at 0x7f58083928c8>], true_wd=True, bn_wd=True, wd=0.01, train_bn=True, path=PosixPath('.'), model_dir='models', callback_fns=[functools.partial(<class 'fastai.basic_train.Recorder'>, add_time=True, silent=False)], callbacks=[...], layer_groups=[Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(400, 1152, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1152, 1152, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1152, 400, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): Embedding(7520, 400, padding_idx=1)\n",
       "  (1): EmbeddingDropout(\n",
       "    (emb): Embedding(7520, 400, padding_idx=1)\n",
       "  )\n",
       "  (2): LinearDecoder(\n",
       "    (decoder): Linear(in_features=400, out_features=7520, bias=True)\n",
       "    (output_dp): RNNDropout()\n",
       "  )\n",
       ")], add_time=True, silent=False)\n",
       "monitor: valid_loss\n",
       "mode: auto\n",
       "min_delta: 0\n",
       "patience: 10, SaveModelCallback\n",
       "learn: LanguageLearner(data=TextLMDataBunch;\n",
       "\n",
       "Train: LabelList (8700 items)\n",
       "x: LMTextList\n",
       "xxbos # xxmaj thorium xxmaj radioactive xxmaj weapons . xxmaj xxunk murders and environmental devastation : - xxup video http : / / t.co / xxunk,xxbos @_minimehh @cjoyner i must be overlooking the burning buildings ? # blacklivesmatter,xxbos xxmaj economic xxmaj collapse xxmaj xxunk : xxmaj specific actions and xxunk to securing lasting wealth from the financial blowout . http : / / t.co / xxunk,xxbos xxunk we will never know what would have happened but the govt seemed to think that their xxunk xxunk the deaths of innocent xxunk,xxbos xxmaj first xxmaj responders xxmaj xxunk for xxmaj national xxmaj summit and xxmaj xxunk on xxup xxunk xxmaj technology http : / / t.co / xxunk # xxunk # xxunk\n",
       "y: LMLabelList\n",
       ",,,,\n",
       "Path: .;\n",
       "\n",
       "Valid: LabelList (2176 items)\n",
       "x: LMTextList\n",
       "xxbos xxmaj xxunk xxmaj xxunk xxmaj health xxmaj care xxmaj reviews xxmaj with xxmaj xxunk xxmaj journalism : xxmaj sick and injured xxunk at a local xxup er are t ... http : / / t.co / xxunk,xxbos xxup lol xxmaj xxunk in the midst of xxunk meltdown reaching for xxmaj double xxmaj xxunk . # xxmaj mets,xxbos xxmaj choking xxmaj hazard xxmaj prompts xxmaj recall xxmaj of xxmaj xxunk xxmaj cheese xxmaj singles http : / / t.co / xxunk,xxbos xxmaj what a night ! xxmaj xxunk go on vacation to end our xxmaj dream xxmaj job party where fire truck xxunk the pool with a xxunk ! http : / / t.co / xxunk,xxbos who said this ? xxmaj xxunk xxmaj sam or xxmaj xxunk ? xxunk xxunk vote for a flattened out xxunk skin a - huh huh xxup û _ i always xxunk\n",
       "y: LMLabelList\n",
       ",,,,\n",
       "Path: .;\n",
       "\n",
       "Test: None, model=SequentialRNN(\n",
       "  (0): AWD_LSTM(\n",
       "    (encoder): Embedding(7520, 400, padding_idx=1)\n",
       "    (encoder_dp): EmbeddingDropout(\n",
       "      (emb): Embedding(7520, 400, padding_idx=1)\n",
       "    )\n",
       "    (rnns): ModuleList(\n",
       "      (0): WeightDropout(\n",
       "        (module): LSTM(400, 1152, batch_first=True)\n",
       "      )\n",
       "      (1): WeightDropout(\n",
       "        (module): LSTM(1152, 1152, batch_first=True)\n",
       "      )\n",
       "      (2): WeightDropout(\n",
       "        (module): LSTM(1152, 400, batch_first=True)\n",
       "      )\n",
       "    )\n",
       "    (input_dp): RNNDropout()\n",
       "    (hidden_dps): ModuleList(\n",
       "      (0): RNNDropout()\n",
       "      (1): RNNDropout()\n",
       "      (2): RNNDropout()\n",
       "    )\n",
       "  )\n",
       "  (1): LinearDecoder(\n",
       "    (decoder): Linear(in_features=400, out_features=7520, bias=True)\n",
       "    (output_dp): RNNDropout()\n",
       "  )\n",
       "), opt_func=functools.partial(<class 'torch.optim.adam.Adam'>, betas=(0.9, 0.99)), loss_func=FlattenedLoss of CrossEntropyLoss(), metrics=[<function accuracy at 0x7f58083928c8>], true_wd=True, bn_wd=True, wd=0.01, train_bn=True, path=PosixPath('.'), model_dir='models', callback_fns=[functools.partial(<class 'fastai.basic_train.Recorder'>, add_time=True, silent=False)], callbacks=[...], layer_groups=[Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(400, 1152, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1152, 1152, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1152, 400, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): Embedding(7520, 400, padding_idx=1)\n",
       "  (1): EmbeddingDropout(\n",
       "    (emb): Embedding(7520, 400, padding_idx=1)\n",
       "  )\n",
       "  (2): LinearDecoder(\n",
       "    (decoder): Linear(in_features=400, out_features=7520, bias=True)\n",
       "    (output_dp): RNNDropout()\n",
       "  )\n",
       ")], add_time=True, silent=False)\n",
       "monitor: accuracy\n",
       "mode: auto\n",
       "every: improvement\n",
       "name: bestmodel, SaveModelCallback\n",
       "learn: LanguageLearner(data=TextLMDataBunch;\n",
       "\n",
       "Train: LabelList (8700 items)\n",
       "x: LMTextList\n",
       "xxbos # xxmaj thorium xxmaj radioactive xxmaj weapons . xxmaj xxunk murders and environmental devastation : - xxup video http : / / t.co / xxunk,xxbos @_minimehh @cjoyner i must be overlooking the burning buildings ? # blacklivesmatter,xxbos xxmaj economic xxmaj collapse xxmaj xxunk : xxmaj specific actions and xxunk to securing lasting wealth from the financial blowout . http : / / t.co / xxunk,xxbos xxunk we will never know what would have happened but the govt seemed to think that their xxunk xxunk the deaths of innocent xxunk,xxbos xxmaj first xxmaj responders xxmaj xxunk for xxmaj national xxmaj summit and xxmaj xxunk on xxup xxunk xxmaj technology http : / / t.co / xxunk # xxunk # xxunk\n",
       "y: LMLabelList\n",
       ",,,,\n",
       "Path: .;\n",
       "\n",
       "Valid: LabelList (2176 items)\n",
       "x: LMTextList\n",
       "xxbos xxmaj xxunk xxmaj xxunk xxmaj health xxmaj care xxmaj reviews xxmaj with xxmaj xxunk xxmaj journalism : xxmaj sick and injured xxunk at a local xxup er are t ... http : / / t.co / xxunk,xxbos xxup lol xxmaj xxunk in the midst of xxunk meltdown reaching for xxmaj double xxmaj xxunk . # xxmaj mets,xxbos xxmaj choking xxmaj hazard xxmaj prompts xxmaj recall xxmaj of xxmaj xxunk xxmaj cheese xxmaj singles http : / / t.co / xxunk,xxbos xxmaj what a night ! xxmaj xxunk go on vacation to end our xxmaj dream xxmaj job party where fire truck xxunk the pool with a xxunk ! http : / / t.co / xxunk,xxbos who said this ? xxmaj xxunk xxmaj sam or xxmaj xxunk ? xxunk xxunk vote for a flattened out xxunk skin a - huh huh xxup û _ i always xxunk\n",
       "y: LMLabelList\n",
       ",,,,\n",
       "Path: .;\n",
       "\n",
       "Test: None, model=SequentialRNN(\n",
       "  (0): AWD_LSTM(\n",
       "    (encoder): Embedding(7520, 400, padding_idx=1)\n",
       "    (encoder_dp): EmbeddingDropout(\n",
       "      (emb): Embedding(7520, 400, padding_idx=1)\n",
       "    )\n",
       "    (rnns): ModuleList(\n",
       "      (0): WeightDropout(\n",
       "        (module): LSTM(400, 1152, batch_first=True)\n",
       "      )\n",
       "      (1): WeightDropout(\n",
       "        (module): LSTM(1152, 1152, batch_first=True)\n",
       "      )\n",
       "      (2): WeightDropout(\n",
       "        (module): LSTM(1152, 400, batch_first=True)\n",
       "      )\n",
       "    )\n",
       "    (input_dp): RNNDropout()\n",
       "    (hidden_dps): ModuleList(\n",
       "      (0): RNNDropout()\n",
       "      (1): RNNDropout()\n",
       "      (2): RNNDropout()\n",
       "    )\n",
       "  )\n",
       "  (1): LinearDecoder(\n",
       "    (decoder): Linear(in_features=400, out_features=7520, bias=True)\n",
       "    (output_dp): RNNDropout()\n",
       "  )\n",
       "), opt_func=functools.partial(<class 'torch.optim.adam.Adam'>, betas=(0.9, 0.99)), loss_func=FlattenedLoss of CrossEntropyLoss(), metrics=[<function accuracy at 0x7f58083928c8>], true_wd=True, bn_wd=True, wd=0.01, train_bn=True, path=PosixPath('.'), model_dir='models', callback_fns=[functools.partial(<class 'fastai.basic_train.Recorder'>, add_time=True, silent=False)], callbacks=[...], layer_groups=[Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(400, 1152, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1152, 1152, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1152, 400, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): Embedding(7520, 400, padding_idx=1)\n",
       "  (1): EmbeddingDropout(\n",
       "    (emb): Embedding(7520, 400, padding_idx=1)\n",
       "  )\n",
       "  (2): LinearDecoder(\n",
       "    (decoder): Linear(in_features=400, out_features=7520, bias=True)\n",
       "    (output_dp): RNNDropout()\n",
       "  )\n",
       ")], add_time=True, silent=False)\n",
       "monitor: accuracy\n",
       "mode: auto\n",
       "every: improvement\n",
       "name: bestmodel, SaveModelCallback\n",
       "learn: LanguageLearner(data=TextLMDataBunch;\n",
       "\n",
       "Train: LabelList (8700 items)\n",
       "x: LMTextList\n",
       "xxbos # xxmaj thorium xxmaj radioactive xxmaj weapons . xxmaj xxunk murders and environmental devastation : - xxup video http : / / t.co / xxunk,xxbos @_minimehh @cjoyner i must be overlooking the burning buildings ? # blacklivesmatter,xxbos xxmaj economic xxmaj collapse xxmaj xxunk : xxmaj specific actions and xxunk to securing lasting wealth from the financial blowout . http : / / t.co / xxunk,xxbos xxunk we will never know what would have happened but the govt seemed to think that their xxunk xxunk the deaths of innocent xxunk,xxbos xxmaj first xxmaj responders xxmaj xxunk for xxmaj national xxmaj summit and xxmaj xxunk on xxup xxunk xxmaj technology http : / / t.co / xxunk # xxunk # xxunk\n",
       "y: LMLabelList\n",
       ",,,,\n",
       "Path: .;\n",
       "\n",
       "Valid: LabelList (2176 items)\n",
       "x: LMTextList\n",
       "xxbos xxmaj xxunk xxmaj xxunk xxmaj health xxmaj care xxmaj reviews xxmaj with xxmaj xxunk xxmaj journalism : xxmaj sick and injured xxunk at a local xxup er are t ... http : / / t.co / xxunk,xxbos xxup lol xxmaj xxunk in the midst of xxunk meltdown reaching for xxmaj double xxmaj xxunk . # xxmaj mets,xxbos xxmaj choking xxmaj hazard xxmaj prompts xxmaj recall xxmaj of xxmaj xxunk xxmaj cheese xxmaj singles http : / / t.co / xxunk,xxbos xxmaj what a night ! xxmaj xxunk go on vacation to end our xxmaj dream xxmaj job party where fire truck xxunk the pool with a xxunk ! http : / / t.co / xxunk,xxbos who said this ? xxmaj xxunk xxmaj sam or xxmaj xxunk ? xxunk xxunk vote for a flattened out xxunk skin a - huh huh xxup û _ i always xxunk\n",
       "y: LMLabelList\n",
       ",,,,\n",
       "Path: .;\n",
       "\n",
       "Test: None, model=SequentialRNN(\n",
       "  (0): AWD_LSTM(\n",
       "    (encoder): Embedding(7520, 400, padding_idx=1)\n",
       "    (encoder_dp): EmbeddingDropout(\n",
       "      (emb): Embedding(7520, 400, padding_idx=1)\n",
       "    )\n",
       "    (rnns): ModuleList(\n",
       "      (0): WeightDropout(\n",
       "        (module): LSTM(400, 1152, batch_first=True)\n",
       "      )\n",
       "      (1): WeightDropout(\n",
       "        (module): LSTM(1152, 1152, batch_first=True)\n",
       "      )\n",
       "      (2): WeightDropout(\n",
       "        (module): LSTM(1152, 400, batch_first=True)\n",
       "      )\n",
       "    )\n",
       "    (input_dp): RNNDropout()\n",
       "    (hidden_dps): ModuleList(\n",
       "      (0): RNNDropout()\n",
       "      (1): RNNDropout()\n",
       "      (2): RNNDropout()\n",
       "    )\n",
       "  )\n",
       "  (1): LinearDecoder(\n",
       "    (decoder): Linear(in_features=400, out_features=7520, bias=True)\n",
       "    (output_dp): RNNDropout()\n",
       "  )\n",
       "), opt_func=functools.partial(<class 'torch.optim.adam.Adam'>, betas=(0.9, 0.99)), loss_func=FlattenedLoss of CrossEntropyLoss(), metrics=[<function accuracy at 0x7f58083928c8>], true_wd=True, bn_wd=True, wd=0.01, train_bn=True, path=PosixPath('.'), model_dir='models', callback_fns=[functools.partial(<class 'fastai.basic_train.Recorder'>, add_time=True, silent=False)], callbacks=[...], layer_groups=[Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(400, 1152, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1152, 1152, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1152, 400, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): Embedding(7520, 400, padding_idx=1)\n",
       "  (1): EmbeddingDropout(\n",
       "    (emb): Embedding(7520, 400, padding_idx=1)\n",
       "  )\n",
       "  (2): LinearDecoder(\n",
       "    (decoder): Linear(in_features=400, out_features=7520, bias=True)\n",
       "    (output_dp): RNNDropout()\n",
       "  )\n",
       ")], add_time=True, silent=False)\n",
       "monitor: accuracy\n",
       "mode: auto\n",
       "every: improvement\n",
       "name: bestmodel], layer_groups=[Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(400, 1152, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1152, 1152, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1152, 400, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): Embedding(7520, 400, padding_idx=1)\n",
       "  (1): EmbeddingDropout(\n",
       "    (emb): Embedding(7520, 400, padding_idx=1)\n",
       "  )\n",
       "  (2): LinearDecoder(\n",
       "    (decoder): Linear(in_features=400, out_features=7520, bias=True)\n",
       "    (output_dp): RNNDropout()\n",
       "  )\n",
       ")], add_time=True, silent=False)"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.load('bestmodel')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save_encoder('ft')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn = None\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data_clas = TextClasDataBunch.from_csv(dir_pth, 'train.csv', vocab=data_lm.train_ds.vocab, bs=32, text_cols=3, label_cols=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_clas.save('data_clas')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RNNLearner(data=TextClasDataBunch;\n",
       "\n",
       "Train: LabelList (6090 items)\n",
       "x: TextList\n",
       "xxbos in xxup both ' xxunk and times of national emergency . ',xxbos xxunk ok i was n't completely xxunk i may have also been in a food xxunk bc of the xxunk / xxunk / xxunk i also annihilated w / xxunk,xxbos xxmaj ladies here 's how to recover from a # date you totally xxup bombed ... according to men http : / / t.co / xxunk http : / / t.co / xxunk,xxbos xxmaj the xxmaj latest : xxmaj more homes razed by xxmaj northern xxmaj california wildfire : xxmaj the latest on wildfires burning in xxmaj california andû _ http : / / t.co / xxunk,xxbos xxmaj it hurts for me to eat cause i burned my xxunk with a xxunk yesterday !\n",
       "y: CategoryList\n",
       "1,0,0,1,0\n",
       "Path: .;\n",
       "\n",
       "Valid: LabelList (1523 items)\n",
       "x: TextList\n",
       "xxbos xxmaj students at xxmaj xxunk remember xxmaj australian casualties at xxmaj xxunk xxmaj xxunk xxmaj gallipoli \n",
       "  http : / / t.co / xxunk via xxunk,xxbos xxmaj israel wrecked my home . xxmaj now it wants my land . \n",
       "  https : / / t.co / xxunk,xxbos ' xxunk : xxmaj haha jam xxunk xxunk garden city xxunk bumper to bumper with xxup xxunk xxunk decide to chill via xxunk ' xxunk siren xxunk,xxbos i feel like i 'm drowning inside my own body ! !,xxbos xxmaj police xxmaj officer xxmaj wounded xxmaj suspect xxmaj dead xxmaj after xxmaj exchanging xxmaj shots : xxmaj richmond police officer wounded suspect killed a ... http : / / t.co / xxunk\n",
       "y: CategoryList\n",
       "1,1,0,0,1\n",
       "Path: .;\n",
       "\n",
       "Test: None, model=SequentialRNN(\n",
       "  (0): MultiBatchEncoder(\n",
       "    (module): AWD_LSTM(\n",
       "      (encoder): Embedding(7520, 400, padding_idx=1)\n",
       "      (encoder_dp): EmbeddingDropout(\n",
       "        (emb): Embedding(7520, 400, padding_idx=1)\n",
       "      )\n",
       "      (rnns): ModuleList(\n",
       "        (0): WeightDropout(\n",
       "          (module): LSTM(400, 1152, batch_first=True)\n",
       "        )\n",
       "        (1): WeightDropout(\n",
       "          (module): LSTM(1152, 1152, batch_first=True)\n",
       "        )\n",
       "        (2): WeightDropout(\n",
       "          (module): LSTM(1152, 400, batch_first=True)\n",
       "        )\n",
       "      )\n",
       "      (input_dp): RNNDropout()\n",
       "      (hidden_dps): ModuleList(\n",
       "        (0): RNNDropout()\n",
       "        (1): RNNDropout()\n",
       "        (2): RNNDropout()\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (1): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.4, inplace=False)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace=True)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1, inplace=False)\n",
       "      (6): Linear(in_features=50, out_features=2, bias=True)\n",
       "    )\n",
       "  )\n",
       "), opt_func=functools.partial(<class 'torch.optim.adam.Adam'>, betas=(0.9, 0.99)), loss_func=FlattenedLoss of CrossEntropyLoss(), metrics=[<function accuracy at 0x7f58083928c8>], true_wd=True, bn_wd=True, wd=0.01, train_bn=True, path=PosixPath('.'), model_dir='models', callback_fns=[functools.partial(<class 'fastai.basic_train.Recorder'>, add_time=True, silent=False)], callbacks=[RNNTrainer\n",
       "learn: RNNLearner(data=TextClasDataBunch;\n",
       "\n",
       "Train: LabelList (6090 items)\n",
       "x: TextList\n",
       "xxbos in xxup both ' xxunk and times of national emergency . ',xxbos xxunk ok i was n't completely xxunk i may have also been in a food xxunk bc of the xxunk / xxunk / xxunk i also annihilated w / xxunk,xxbos xxmaj ladies here 's how to recover from a # date you totally xxup bombed ... according to men http : / / t.co / xxunk http : / / t.co / xxunk,xxbos xxmaj the xxmaj latest : xxmaj more homes razed by xxmaj northern xxmaj california wildfire : xxmaj the latest on wildfires burning in xxmaj california andû _ http : / / t.co / xxunk,xxbos xxmaj it hurts for me to eat cause i burned my xxunk with a xxunk yesterday !\n",
       "y: CategoryList\n",
       "1,0,0,1,0\n",
       "Path: .;\n",
       "\n",
       "Valid: LabelList (1523 items)\n",
       "x: TextList\n",
       "xxbos xxmaj students at xxmaj xxunk remember xxmaj australian casualties at xxmaj xxunk xxmaj xxunk xxmaj gallipoli \n",
       "  http : / / t.co / xxunk via xxunk,xxbos xxmaj israel wrecked my home . xxmaj now it wants my land . \n",
       "  https : / / t.co / xxunk,xxbos ' xxunk : xxmaj haha jam xxunk xxunk garden city xxunk bumper to bumper with xxup xxunk xxunk decide to chill via xxunk ' xxunk siren xxunk,xxbos i feel like i 'm drowning inside my own body ! !,xxbos xxmaj police xxmaj officer xxmaj wounded xxmaj suspect xxmaj dead xxmaj after xxmaj exchanging xxmaj shots : xxmaj richmond police officer wounded suspect killed a ... http : / / t.co / xxunk\n",
       "y: CategoryList\n",
       "1,1,0,0,1\n",
       "Path: .;\n",
       "\n",
       "Test: None, model=SequentialRNN(\n",
       "  (0): MultiBatchEncoder(\n",
       "    (module): AWD_LSTM(\n",
       "      (encoder): Embedding(7520, 400, padding_idx=1)\n",
       "      (encoder_dp): EmbeddingDropout(\n",
       "        (emb): Embedding(7520, 400, padding_idx=1)\n",
       "      )\n",
       "      (rnns): ModuleList(\n",
       "        (0): WeightDropout(\n",
       "          (module): LSTM(400, 1152, batch_first=True)\n",
       "        )\n",
       "        (1): WeightDropout(\n",
       "          (module): LSTM(1152, 1152, batch_first=True)\n",
       "        )\n",
       "        (2): WeightDropout(\n",
       "          (module): LSTM(1152, 400, batch_first=True)\n",
       "        )\n",
       "      )\n",
       "      (input_dp): RNNDropout()\n",
       "      (hidden_dps): ModuleList(\n",
       "        (0): RNNDropout()\n",
       "        (1): RNNDropout()\n",
       "        (2): RNNDropout()\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (1): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.4, inplace=False)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace=True)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1, inplace=False)\n",
       "      (6): Linear(in_features=50, out_features=2, bias=True)\n",
       "    )\n",
       "  )\n",
       "), opt_func=functools.partial(<class 'torch.optim.adam.Adam'>, betas=(0.9, 0.99)), loss_func=FlattenedLoss of CrossEntropyLoss(), metrics=[<function accuracy at 0x7f58083928c8>], true_wd=True, bn_wd=True, wd=0.01, train_bn=True, path=PosixPath('.'), model_dir='models', callback_fns=[functools.partial(<class 'fastai.basic_train.Recorder'>, add_time=True, silent=False)], callbacks=[...], layer_groups=[Sequential(\n",
       "  (0): Embedding(7520, 400, padding_idx=1)\n",
       "  (1): EmbeddingDropout(\n",
       "    (emb): Embedding(7520, 400, padding_idx=1)\n",
       "  )\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(400, 1152, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1152, 1152, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1152, 400, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.4, inplace=False)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace=True)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1, inplace=False)\n",
       "      (6): Linear(in_features=50, out_features=2, bias=True)\n",
       "    )\n",
       "  )\n",
       ")], add_time=True, silent=False)\n",
       "alpha: 2.0\n",
       "beta: 1.0, SaveModelCallback\n",
       "learn: RNNLearner(data=TextClasDataBunch;\n",
       "\n",
       "Train: LabelList (6090 items)\n",
       "x: TextList\n",
       "xxbos in xxup both ' xxunk and times of national emergency . ',xxbos xxunk ok i was n't completely xxunk i may have also been in a food xxunk bc of the xxunk / xxunk / xxunk i also annihilated w / xxunk,xxbos xxmaj ladies here 's how to recover from a # date you totally xxup bombed ... according to men http : / / t.co / xxunk http : / / t.co / xxunk,xxbos xxmaj the xxmaj latest : xxmaj more homes razed by xxmaj northern xxmaj california wildfire : xxmaj the latest on wildfires burning in xxmaj california andû _ http : / / t.co / xxunk,xxbos xxmaj it hurts for me to eat cause i burned my xxunk with a xxunk yesterday !\n",
       "y: CategoryList\n",
       "1,0,0,1,0\n",
       "Path: .;\n",
       "\n",
       "Valid: LabelList (1523 items)\n",
       "x: TextList\n",
       "xxbos xxmaj students at xxmaj xxunk remember xxmaj australian casualties at xxmaj xxunk xxmaj xxunk xxmaj gallipoli \n",
       "  http : / / t.co / xxunk via xxunk,xxbos xxmaj israel wrecked my home . xxmaj now it wants my land . \n",
       "  https : / / t.co / xxunk,xxbos ' xxunk : xxmaj haha jam xxunk xxunk garden city xxunk bumper to bumper with xxup xxunk xxunk decide to chill via xxunk ' xxunk siren xxunk,xxbos i feel like i 'm drowning inside my own body ! !,xxbos xxmaj police xxmaj officer xxmaj wounded xxmaj suspect xxmaj dead xxmaj after xxmaj exchanging xxmaj shots : xxmaj richmond police officer wounded suspect killed a ... http : / / t.co / xxunk\n",
       "y: CategoryList\n",
       "1,1,0,0,1\n",
       "Path: .;\n",
       "\n",
       "Test: None, model=SequentialRNN(\n",
       "  (0): MultiBatchEncoder(\n",
       "    (module): AWD_LSTM(\n",
       "      (encoder): Embedding(7520, 400, padding_idx=1)\n",
       "      (encoder_dp): EmbeddingDropout(\n",
       "        (emb): Embedding(7520, 400, padding_idx=1)\n",
       "      )\n",
       "      (rnns): ModuleList(\n",
       "        (0): WeightDropout(\n",
       "          (module): LSTM(400, 1152, batch_first=True)\n",
       "        )\n",
       "        (1): WeightDropout(\n",
       "          (module): LSTM(1152, 1152, batch_first=True)\n",
       "        )\n",
       "        (2): WeightDropout(\n",
       "          (module): LSTM(1152, 400, batch_first=True)\n",
       "        )\n",
       "      )\n",
       "      (input_dp): RNNDropout()\n",
       "      (hidden_dps): ModuleList(\n",
       "        (0): RNNDropout()\n",
       "        (1): RNNDropout()\n",
       "        (2): RNNDropout()\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (1): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.4, inplace=False)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace=True)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1, inplace=False)\n",
       "      (6): Linear(in_features=50, out_features=2, bias=True)\n",
       "    )\n",
       "  )\n",
       "), opt_func=functools.partial(<class 'torch.optim.adam.Adam'>, betas=(0.9, 0.99)), loss_func=FlattenedLoss of CrossEntropyLoss(), metrics=[<function accuracy at 0x7f58083928c8>], true_wd=True, bn_wd=True, wd=0.01, train_bn=True, path=PosixPath('.'), model_dir='models', callback_fns=[functools.partial(<class 'fastai.basic_train.Recorder'>, add_time=True, silent=False)], callbacks=[...], layer_groups=[Sequential(\n",
       "  (0): Embedding(7520, 400, padding_idx=1)\n",
       "  (1): EmbeddingDropout(\n",
       "    (emb): Embedding(7520, 400, padding_idx=1)\n",
       "  )\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(400, 1152, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1152, 1152, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1152, 400, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.4, inplace=False)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace=True)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1, inplace=False)\n",
       "      (6): Linear(in_features=50, out_features=2, bias=True)\n",
       "    )\n",
       "  )\n",
       ")], add_time=True, silent=False)\n",
       "monitor: accuracy\n",
       "mode: auto\n",
       "every: improvement\n",
       "name: best_clas], layer_groups=[Sequential(\n",
       "  (0): Embedding(7520, 400, padding_idx=1)\n",
       "  (1): EmbeddingDropout(\n",
       "    (emb): Embedding(7520, 400, padding_idx=1)\n",
       "  )\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(400, 1152, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1152, 1152, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1152, 400, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.4, inplace=False)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace=True)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1, inplace=False)\n",
       "      (6): Linear(in_features=50, out_features=2, bias=True)\n",
       "    )\n",
       "  )\n",
       ")], add_time=True, silent=False)"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn = text_classifier_learner(data_clas, AWD_LSTM, drop_mult=1.0)\n",
    "learn.callbacks.append(SaveModelCallback(learn, monitor='accuracy', name = 'best_clas'))\n",
    "learn.load_encoder('ft')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>xxbos _ \\n  xxrep 5 ? xxup retweet \\n  xxrep 7 ? \\n  xxrep 5 ? xxup follow xxup all xxup who xxup rt \\n  xxrep 7 ? \\n  xxrep 5 ? xxup followback \\n  xxrep 7 ? \\n  xxrep 5 ? xxup gain xxup with \\n  xxrep 7 ? \\n  xxrep 5 ? xxup follow ? xxunk # xxup xxunk</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>xxbos xxup info xxup u. xxup cld : xxup xxunk xxup xxunk . xxup exp xxup inst xxup apch . xxup rwy 05 . xxup curfew xxup in xxup oper xxup until 2030 xxup z. xxup taxiways xxup foxtrot 5 &amp; &amp; xxup foxtrot 6 xxup navbl . xxup tmp : 10 . xxup wnd : 030 / 6 .</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>xxbos xxmaj truth ... \\n  https : / / t.co / xxunk \\n  # xxmaj news \\n  # xxup bbc \\n  # xxup cnn \\n  # xxmaj islam \\n  # xxmaj truth \\n  # god \\n  # xxup isis \\n  # terrorism \\n  # xxmaj quran \\n  # xxmaj lies http : / / t.co / xxunk</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>xxbos xxmaj no # news of # hostages in # xxmaj libya \\n \\n  http : / / t.co / xxunk \\n \\n  # xxmaj india # terrorism # xxmaj africa # xxup ap # xxup ts # xxup nri # xxmaj news # xxup trs # xxup tdp # xxup bjp http : / / t.co / xxunk</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>xxbos xxmaj truth ... \\n  https : / / t.co / xxunk \\n  # xxmaj news \\n  # xxup bbc \\n  # xxup cnn \\n  # xxmaj islam \\n  # xxmaj truth \\n  # god \\n  # xxup isis \\n  # terrorism \\n  # xxmaj quran \\n  # xxmaj lies http : / / t.co / xxunk</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data_clas.show_batch()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.575458</td>\n",
       "      <td>0.464918</td>\n",
       "      <td>0.783979</td>\n",
       "      <td>00:09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.583548</td>\n",
       "      <td>0.465101</td>\n",
       "      <td>0.791202</td>\n",
       "      <td>00:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.570289</td>\n",
       "      <td>0.481283</td>\n",
       "      <td>0.774130</td>\n",
       "      <td>00:10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.570137</td>\n",
       "      <td>0.502807</td>\n",
       "      <td>0.776756</td>\n",
       "      <td>00:10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.571827</td>\n",
       "      <td>0.456672</td>\n",
       "      <td>0.793171</td>\n",
       "      <td>00:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>0.577637</td>\n",
       "      <td>0.473508</td>\n",
       "      <td>0.794485</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>0.555232</td>\n",
       "      <td>0.458774</td>\n",
       "      <td>0.802364</td>\n",
       "      <td>00:16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>0.557069</td>\n",
       "      <td>0.460871</td>\n",
       "      <td>0.793828</td>\n",
       "      <td>00:17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>0.547261</td>\n",
       "      <td>0.449864</td>\n",
       "      <td>0.793828</td>\n",
       "      <td>00:19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>0.541085</td>\n",
       "      <td>0.447655</td>\n",
       "      <td>0.795141</td>\n",
       "      <td>00:18</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Better model found at epoch 0 with accuracy value: 0.7839789986610413.\n",
      "Better model found at epoch 1 with accuracy value: 0.7912015914916992.\n",
      "Better model found at epoch 4 with accuracy value: 0.7931713461875916.\n",
      "Better model found at epoch 5 with accuracy value: 0.794484555721283.\n",
      "Better model found at epoch 6 with accuracy value: 0.8023637533187866.\n"
     ]
    }
   ],
   "source": [
    "learn.fit_one_cycle(10, 1e-2, wd=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.load('best_clas')\n",
    "learn.unfreeze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.546915</td>\n",
       "      <td>0.451598</td>\n",
       "      <td>0.799081</td>\n",
       "      <td>00:31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.528329</td>\n",
       "      <td>0.439921</td>\n",
       "      <td>0.797111</td>\n",
       "      <td>00:46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.509553</td>\n",
       "      <td>0.435978</td>\n",
       "      <td>0.794485</td>\n",
       "      <td>00:43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.499340</td>\n",
       "      <td>0.428041</td>\n",
       "      <td>0.810243</td>\n",
       "      <td>00:45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.483430</td>\n",
       "      <td>0.424552</td>\n",
       "      <td>0.807617</td>\n",
       "      <td>00:47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>0.489588</td>\n",
       "      <td>0.424988</td>\n",
       "      <td>0.809586</td>\n",
       "      <td>00:47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>0.454515</td>\n",
       "      <td>0.421418</td>\n",
       "      <td>0.809586</td>\n",
       "      <td>00:48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>0.452988</td>\n",
       "      <td>0.420954</td>\n",
       "      <td>0.808930</td>\n",
       "      <td>00:44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>0.440374</td>\n",
       "      <td>0.424487</td>\n",
       "      <td>0.809586</td>\n",
       "      <td>00:48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>0.442439</td>\n",
       "      <td>0.420062</td>\n",
       "      <td>0.807617</td>\n",
       "      <td>00:45</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Better model found at epoch 0 with accuracy value: 0.7990807890892029.\n",
      "Better model found at epoch 3 with accuracy value: 0.8102429509162903.\n"
     ]
    }
   ],
   "source": [
    "learn.fit_one_cycle(10, slice(2e-3/100, 2e-3), wd = 0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEGCAYAAABy53LJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOydeXhV1dWH35V5JiMyBAggyDyEiAOi4ohiwVmpQ7Gf0upnrfXTinUsVkVtrdVaW21FbVW0WBUriqjgPBCUQeYwScIUwpCQhIz7++Oce++5U3Iz3CSE9T7PeXLOPnvvu89Fz+/uvdZeS4wxKIqiKEqoRLT3ABRFUZTDCxUORVEUpUmocCiKoihNQoVDURRFaRIqHIqiKEqTiGrvAbQFmZmZJicnp72HoSiKclixdOnSPcaYLN/yI0I4cnJyyM/Pb+9hKIqiHFaIyNZA5bpUpSiKojQJFQ5FURSlSYRVOERkooisE5ECEZkRpM6lIrJaRFaJyMt22SgR+dIuWyEilznqPy8im0VkmX2MCuczKIqiKN6EzcYhIpHAU8CZQCGwRETmGWNWO+oMAO4Axhlj9olIV/tWBXC1MWaDiPQAlorIAmPMfvv+bcaYueEau6IoHZOamhoKCws5dOhQew+lUxEXF0d2djbR0dEh1Q+ncXwsUGCM2QQgInOAKcBqR53rgKeMMfsAjDG77b/rXRWMMdtFZDeQBexHUZQjlsLCQpKTk8nJyUFE2ns4nQJjDCUlJRQWFtK3b9+Q2oRzqaonsM1xXWiXORkIDBSRz0XkKxGZ6NuJiIwFYoCNjuIH7CWsP4pIbKAPF5HpIpIvIvnFxcUtexJFUToEhw4dIiMjQ0WjFRERMjIymjSLa2/jeBQwADgVmAo8KyKprpsi0h34J3CNMabeLr4DGAQcC6QDtwfq2BjzjDEmzxiTl5Xl54asKMphiopG69PU7zScwlEE9HJcZ9tlTgqBecaYGmPMZmA9lpAgIinAO8CdxpivXA2MMTuMRRUwG2tJLCwU7qtg0brd4epeURTlsCScwrEEGCAifUUkBrgcmOdT502s2QYikom1dLXJrv8G8KKvEdyehSCWRJ4PfB+uBzjn8U+5ZvaScHWvKMphRklJCaNGjWLUqFF069aNnj17uq+rq6tD6uOaa65h3bp1YR5peAmbcdwYUysiNwILgEjgOWPMKhGZCeQbY+bZ984SkdVAHZa3VImIXAmcDGSIyDS7y2nGmGXASyKSBQiwDPh5uJ6hrKo2XF0rinIYkpGRwbJlywC47777SEpK4tZbb/WqY4zBGENERODf5bNnzw77OMNNWG0cxpj5xpiBxpj+xpgH7LJ7bNHAXnK6xRgzxBgz3Bgzxy7/lzEm2hgzynEss++dZtcdZoy50hhzMJzPoCiK0hgFBQUMGTKEK664gqFDh7Jjxw6mT59OXl4eQ4cOZebMme66J510EsuWLaO2tpbU1FRmzJjByJEjOeGEE9i9+/BYGj8iYlUpitL5+O3bq1i9vbRV+xzSI4V7fzS0WW3Xrl3Liy++SF5eHgCzZs0iPT2d2tpaJkyYwMUXX8yQIUO82hw4cIBTTjmFWbNmccstt/Dcc88xY0bAvdIdivb2qjosqK/XvOyKojRM//793aIB8Morr5Cbm0tubi5r1qxh9erVfm3i4+M555xzABgzZgxbtmxpq+G2CJ1xhECdMUSgLoCK0pFo7swgXCQmJrrPN2zYwJ/+9Ce++eYbUlNTufLKKwPuk4iJiXGfR0ZGUlt7eNhVdcYRAnU641AUpQmUlpaSnJxMSkoKO3bsYMGCBe09pFZFZxwhUG9UOBRFCZ3c3FyGDBnCoEGD6NOnD+PGjWvvIbUqYo6Al2JeXp5pTiKnnBnvALDivrNIiQst+JeiKOFjzZo1DB48uL2H0SkJ9N2KyFJjTJ5vXV2qCgE1jiuKonhQ4QgBtXEoiqJ4UOEIgbojYDlPURQlVFQ4GiDC9sCtr2+4nqIoypGECkcDRNrKUavKoSiK4kaFowFcMepVNxRFUTyocDSAa6lKbRyKogBMmDDBbzPf448/zvXXXx+0TVJSEgDbt2/n4osvDljn1FNPpbEtA48//jgVFRXu63PPPZf9+9snm7YKRwNE2DMO9apSFAVg6tSpzJkzx6tszpw5TJ06tdG2PXr0YO7cuY3WC4avcMyfP5/U1NQGWoQPFY4GiFThUBTFwcUXX8w777zjTtq0ZcsWtm/fzujRozn99NPJzc1l+PDhvPXWW35tt2zZwrBhwwCorKzk8ssvZ/DgwVxwwQVUVla6611//fXucOz33nsvAE888QTbt29nwoQJTJgwAYCcnBz27NkDwGOPPcawYcMYNmwYjz/+uPvzBg8ezHXXXcfQoUM566yzvD6nJYQ15IiITAT+hJXI6e/GmFkB6lwK3AcYYLkx5sd2+U+Au+xqvzPGvGCXjwGeB+KB+cAvTZi2v0dEqHAoSofl3Rmwc2Xr9tltOJzj95pyk56eztixY3n33XeZMmUKc+bM4dJLLyU+Pp433niDlJQU9uzZw/HHH8/kyZOD5vJ++umnSUhIYM2aNaxYsYLc3Fz3vQceeID09HTq6uo4/fTTWbFiBTfddBOPPfYYixYtIjMz06uvpUuXMnv2bL7++muMMRx33HGccsoppKWlsWHDBl555RWeffZZLr30Ul5//XWuvPLKFn9NYZtxiEgk8BRwDjAEmCoiQ3zqDADuAMYZY4YCN9vl6cC9wHFYOcXvFZE0u9nTwHVYuckHABPD9Qxud1y1cSiKYuNcrnItUxlj+M1vfsOIESM444wzKCoqYteuXUH7+OSTT9wv8BEjRjBixAj3vddee43c3FxGjx7NqlWrAoZjd/LZZ59xwQUXkJiYSFJSEhdeeCGffvopAH379mXUqFFA64ZtD+eMYyxQYIzZBCAic4ApgPNbuA54yhizD8AY40p/dTaw0Biz1267EJgoIouBFGPMV3b5i1h5x98NxwOojUNROjANzAzCyZQpU/jVr37Ft99+S0VFBWPGjOH555+nuLiYpUuXEh0dTU5OTsAw6o2xefNmfv/737NkyRLS0tKYNm1as/pxERsb6z6PjIxstaWqcNo4egLbHNeFdpmTgcBAEflcRL6yl7YaatvTPm+oz1bDNc1UrypFUVwkJSUxYcIEfvrTn7qN4gcOHKBr165ER0ezaNEitm7d2mAfJ598Mi+//DIA33//PStWrACscOyJiYl06dKFXbt28e67nt/EycnJlJWV+fU1fvx43nzzTSoqKigvL+eNN95g/PjxrfW4AWnvsOpRWMtNpwLZwCciMrw1OhaR6cB0gN69ezerD7c7rs44FEVxMHXqVC644AL3ktUVV1zBj370I4YPH05eXh6DBg1qsP3111/PNddcw+DBgxk8eDBjxowBYOTIkYwePZpBgwbRq1cvr3Ds06dPZ+LEifTo0YNFixa5y3Nzc5k2bRpjx44F4Nprr2X06NFhzSYYtrDqInICcJ8x5mz7+g4AY8xDjjp/Bb42xsy2rz8EZgBHA6caY35ml/8NWGwfi4wxg+zyqc56wWhuWPUTHvqQHQcOMWf68RzfL6PJ7RVFaV00rHr46Chh1ZcAA0Skr4jEAJcD83zqvIk120BEMrGWrjYBC4CzRCTNNoqfBSwwxuwASkXkeLHWka4G/P3eWokI985xnXEoiqK4CNtSlTGmVkRuxBKBSOA5Y8wqEZkJ5Btj5uERiNVAHXCbMaYEQETuxxIfgJkuQzlwAx533HcJk2HcGoP1V20ciqIoHsJq4zDGzMfaa+Esu8dxboBb7MO37XPAcwHK84FhrT7YAHiCHKpwKEpHwRgTdH+E0jyaarLQneMNEKlLVYrSoYiLi6OkpKTJLzolOMYYSkpKiIuLC7lNe3tVdWhEvaoUpUORnZ1NYWEhxcXF7T2UTkVcXBzZ2dkh11fhaAC3cVx/3ShKhyA6Opq+ffu29zCOeHSpqgE8O8fbeSCKoigdCBWOBnAtVWkGQEVRFA8qHA3g8qrSpSpFURQPKhwNoEtViqIo/qhwNIA7rLp6VSmKorhR4WiACN0AqCiK4ocKRwO4NgD+5o2VlFfVtvNoFEVROgYqHA0Q4QhrULivdRKgKIqiHO6ocDSAy6sKoLpWLeSKoiigwtEgibGejfVVtXXtOBJFUZSOgwpHiFTpjENRFAVQ4WgEjzeVbgJUFEWxUOFoAKdWqEeuoiiKRViFQ0Qmisg6ESkQkRkB7k8TkWIRWWYf19rlExxly0TkkIicb997XkQ2O+6NCtf4nVpRq9vHFUVRgDCGVReRSOAp4EygEFgiIvOMMat9qr5qjLnRWWCMWQSMsvtJBwqA9x1VbjPGzA3X2B3jcJ/X1OmUQ1EUBcI74xgLFBhjNhljqoE5wJRm9HMx8K4xpqJVR9dENEKuoiiKRTiFoyewzXFdaJf5cpGIrBCRuSLSK8D9y4FXfMoesNv8UURiA324iEwXkXwRyW9utjDvpSqdcSiKokD7G8ffBnKMMSOAhcALzpsi0h0YDixwFN8BDAKOBdKB2wN1bIx5xhiTZ4zJy8rKatbg7j5viPu8Rm0ciqIoQHiFowhwziCy7TI3xpgSY0yVffl3YIxPH5cCbxhjahxtdhiLKmA21pJYWOifleQ+10CHiqIoFuEUjiXAABHpKyIxWEtO85wV7BmFi8nAGp8+puKzTOVqIyICnA9838rj9uLUY6zZinpVKYqiWIRNOIwxtcCNWMtMa4DXjDGrRGSmiEy2q90kIqtEZDlwEzDN1V5EcrBmLB/7dP2SiKwEVgKZwO/C9QwAj19mefuqV5WiKIqFmCNgR3ReXp7Jz89vVtuDVbUMu3cBw3t2ISpSeOOGca08OkVRlI6JiCw1xuT5lodtH0dnIcqOkLuy6ABgLVlFRba3T4GiKEr7oW/ARoj2EYlqtXUoinKEo8LRCM6cHKB5ORRFUVQ4mogKh6IoRzoqHE1E83IoinKko8LRRNbtLGvvISiKorQrKhxN5NoX83lrWVHjFVsRY4xuQFQUpcOgwtEMVm0vbdPPu2/eKo6+813qNeyJoigdABWOZiCNV2lVXvhyKwB7DlY1UlNRFCX8qHA0g/YKeLivoqbxSoqiKGFGhaMZpCfGtMvn7i2vbpfPVRRFcaLC0Qxi2jDkiDOW2P4KFQ5FUdofFY5mcKimrs0+62BVrfu8RGcciqJ0AFQ4QuC+Hw1h8sge7uvy6rYRjndX7mD4fe+7r3eVHmqTz1UURWkIFY4QmDauL09MHe2+/uvHG6lrAwP59S9963X95EcFrN3Ztq7AiqIovoRVOERkooisE5ECEZkR4P40ESkWkWX2ca3jXp2jfJ6jvK+IfG33+aqdXbDNcYVZby0qq+tCsmFMfPzTVv1cRVGUphI24RCRSOAp4BxgCDBVRIYEqPqqMWaUffzdUV7pKJ/sKH8Y+KMx5mhgH/A/4XqGhtjXyobq8578lFEzF7qvfTf7xUZ5/ql0F7miKO1JOGccY4ECY8wmY0w1MAeY0pIO7TzjpwFz7aIXsPKOtznlDqN1a7CxuByAX89dzlOLCljrExPryuP7uM8v/uuX/OH9da36+YqiKKESTuHoCWxzXBfaZb5cJCIrRGSuiPRylMeJSL6IfCUiLnHIAPbb+cwb6jPsfLN5b4var95eyiPvrcU3de9r+YU8umAd5z7hWZLKTIrh1rOO4ZIx2QAs27afJz8q4MkPN7iXtyrbyGCvKIrS3qlj3wZeMcZUicjPsGYQp9n3+hhjikSkH/CRiKwEQjYsiMh0YDpA7969W2WwE47JYmiPLvx5UUGLw47c/voKVhYdID46kumn9Ata7y9X5HLmkKOIjoxgyqie/HtpofveHxauZ3dZFZERwvNfbGHuz08gLyedN74r5IM1u3nqx7ktHKWiKIo/4ZxxFAHOGUS2XebGGFNijHEFYPo7MMZxr8j+uwlYDIwGSoBUEXEJnl+fjvbPGGPyjDF5WVlZLX8aYPY1Y7n17GPITIqlaH8lA+6cz/Jt+5vVl8u19g8L13PMXe8FrSd40tdmp8X73X/zuyKe/2ILAE98VADAr15dzjsrdvjNZhRFUVqDcArHEmCA7QUVA1wOzHNWEJHujsvJwBq7PE1EYu3zTGAcsNpYb8JFwMV2m58Ab4XxGQISEyl8sGY3NXWGF+0AhE1ld1loAQu7JES7z3MyE/3ulzlsLZ+sL+aAI57Vrf9ewecFe5o1PkVRlGCEbanKGFMrIjcCC4BI4DljzCoRmQnkG2PmATeJyGSgFtgLTLObDwb+JiL1WOI2yxiz2r53OzBHRH4HfAf8I1zPEIxoh4dTXHTra+/nM05jc3E5InBi/0yve2/ccCIX/OWLoG1PmPWh+/z1bwt5/dtCNj54rl/udEVRlOYSVhuHMWY+MN+n7B7H+R3AHQHafQEMD9LnJiyPrXbDuQIUFx3Z5PaNhSzp0SWOnqn+y1IAo3un8bNT+vG3jzcFvF8RwEj+9OICbjxtQJPHqSiKEgjdOd4Mfthb4T5fvG53k9sX7D7Y4H3L6zg4t589yK4X2uct2bIvtIqKoighoMLRQlz7LwJRuK+CfQECExbuqwTgjnMGNeszIyKEz2ecxvf3nc0jF49g7f0TGZHdJWj9xNimz4oURVGCocLRDKaO7dV4JeCkhxcx/pFFfuUV1ZZBe2zfdL97J/bPCKnvnqnxJMZGcWleL+KiI/nDJSP96pwxuCsAGYmxIfWpKIoSCioczeCB873NLw3lAj8YYIe5a9d5dloCd5wziDMGH+W+N3PKsGaNqVd6gvt8y6xJfP/bs/nbVXn0y0qkpLz1U84W7C7jd/9drXnQFeUIpL03AB6WRPh4KJVX1/LD3greX7WLm88YwBvfFfHAO2uCtr/7rVUAJMVG8bNT+pOeuI0P1uziquP7cHTXpGaNKS46kjvPHUzXlFh33wCZSbHsOdj6eTx+8twSivZXctUJfeiT4e8mrChK50WFo5m8fv2J/HfFdmZ/voXSQ7Vc8JcvqK6t52en9OOW15YHbVdV6/F6crnyXpSbzf6KGi7IbVn0lOtO9t+BnpkUwzqfuFetges5ivZVqnAoyhGGLlU1kzF90pg03Nq/uLJwP9W1VsTafY4NeE6MMewuO8QfF25wl7m8pyIihOtO7kdmUuvbInaXVrGxuJzr/7WUmlaMqpscZ21MfPyDDY3UVBSls6EzjhYwqHsKAFtLPO65gbyovti4hx8/+3WbjcvJmD5p5G/dx7vf7+T9VbuYNKJ7441CICHG8tT6ZstenlpUwFlDjmLAUcmt0reiKB0bnXG0gDh7B3lVbT0x9vn+ADOOQKKxZdak8A7O5sLcbPd5a6aedW5ifHTBOn789/YRRkVR2h4VjhYQFRlBVIRQVVvn/gX+ypIf2nlU3nRPjXOfH6ptndDrRfsr2ba30qus7FDgJTpFUTofKhwtpLbe8NSije6ZxjsrdgSt+/BFlhvvrWcNbJOxAaTEeYIkVtW0jo1j3KyPqPaxl0SEuo1dUZTDHhWOMPKjkT28ri87tjdbZk1q87hRLsFqjRnH7jLPcpdzt7or9LuiKJ0f/b+9hRzfz7P7u68j7PnUsb146ELPRsFXrju+Tcfl5LJje5OWEM02O8bW0q37ePaTwEESL3/mS6Y+85X7eueBQ155PcY+4Im+O7SHCoeiHIno/+0tZM70E9znIx2/wH9x2gASYzwxovpnte9eh7SEGHeww4ue/oIH5ns2KK7ZUcprS6wsv19t2suXm0o4/6nPyZnxDsc/9CH/tZfffAM6pjlyhcRFR1BTV9/qudgVRel4qHC0IovXF7vPu3eJ84pymxGGPRpN4bh+GRSXVTH2gQ/cZbW2neKcP33Kr19f4Y6hBVZecxe/eOU7AKbNXuLVZ15Omvv8UE0d18xewtB7F4Rl/IqidBxC2schIv2BQjs3+KnACOBFY0zz8qZ2UvZX1LD83rOIi47wC43e3omUspIt4XJmHpzxn5XcPtEToTf3/oVB2zuXq+4/fxiXjMmm1OFJtedgNZ/Z2QYP1dQ1K0+JoiiHB6HOOF4H6kTkaOAZrFziLzfWSEQmisg6ESkQkRkB7k8TkWIRWWYf19rlo0TkSxFZJSIrROQyR5vnRWSzo82oEJ8hbLxxw4kA/PGykXSJjyY2quO9NJ3LSi7mLi3k2Ac+cC+pHWrA68qZg6RLfDRx0ZGkxscErFu4ryJguaIonYNQd47X26lgLwCeNMY8KSLfNdRARCKBp4AzgUJgiYjMc6SAdfGqMeZGn7IK4GpjzAYR6QEsFZEFjhnObcaYuSGOPeyM7p0WdENf/l1ndIgIssN7Bs/XERUZATTscbXWEe/qODscfExUBFPH9uKUgV35+b+Wuu+/umQbd04a0rIBK4rSYQl1xlEjIlOBnwD/tcv8f8J6MxYoMMZsMsZUA3OAKaF8mDFmvTFmg32+HdgNZIU41g5FZlIsXVPiGq8YZob0SAlYHh0pHKis4afj+jbYfu0OSzhe+9kJHOV4nocuHMHEYd286j776WY+cdh7FEXpXIQqHNcAJwAPGGM2i0hf4J+NtOkJbHNcF9plvlxkL0fNFRG/DEkiMhaIATY6ih+w2/xRRAJanUVkuojki0h+cbG+xBJiovjbVWO8yvpnJVJTZ82GzhxyFOMHZNInI4G5Pz+B0wd19aq7frclHCnxgSepmUney1YbGkmPqyjK4UtIwmGMWW2MuckY84qIpAHJxpiHW+Hz3wZyjDEjgIXAC86bItIdS6CuMca4FuDvAAYBxwLpwO1BxvyMMSbPGJOXlXVYTlZanbOHdnMbydfMnMiJ/TPd9/plJfLCNWNZ9H+nkpeTzj+mHesO+w6eHfGuPB++nDLQW2icxnRFUToXIQmHiCwWkRQRSQe+BZ4VkccaaVaEZUR3kW2XuTHGlBhjXG4+fwfcP4lFJAV4B7jTGPOVo80OY1EFzMZaElNCZOGvTubD/zuF+JhIkuI8IpAYG0VEhHglqfrt5KHu4I0ukmMDr1A+cMEwXr/+RDY+eC4iUFqpsasUpbMS6lJVF2NMKXAhlhvuccAZjbRZAgwQkb4iEgNcDsxzVrBnFC4mA2vs8hjgDfuz5gZqI5a/6/nA9yE+gwKkJsTQP8vKMpjsEI6EAO6zlx3bm/W/O8erLDE2sMdYXHQkY/qkERkhpMZHs7ei9bMOKorSMQjVqyrKfmFfCtwZSgPbC+tGYAEQCTxnjFklIjOBfGPMPOAmEZkM1AJ7gWl280uBk4EMEXGVTTPGLANeEpEsQIBlwM9DfAbFB1cypkifmYYv3bvEseOAFaMqKoTQImkJMUETWh1uLF63GxHhlIG63KkoLkIVjplYAvC5MWaJiPQDGk39ZoyZD8z3KbvHcX4Hls3Ct92/gH8F6fO0EMesNEK8PcsY3L3hBExv/e84xj74YYN1nKQmRLO/HWYcdfWm1TdaunbLt1X+FEU5HAjVOP5vY8wIY8z19vUmY8xF4R2aEm7y+qSRk5HAfT8a2mA9l0E9VOJjIvm8oIRnPtnYeOVW4vuiA/T/zXw+bkU34CVb9rZaX4rSmQg15Eg28CQwzi76FPilMaYwXANTwk9OZiKLb5vQaD0R4ZvfnB5yv3X2hscH568lt3caeTnpjbRoOa7YWu99v7PVlpX+nb+t8UqKcgQSqnF8NpZhu4d9vG2XKUcIXVPiQt7I6PTEvfivX7rPr30hn9N+v9jtqrvnYBU5M97hwzW7Wjy+vXau92DRef+dv42Fq5v2OaWVnr6qWil7oqJ0BkIVjixjzGxjTK19PM9hupNbaT8+WLOLTXvKecbOBbJ0qxXm/Z9fbW1x348tXA/A5j3lgLV09dxnmwErCvBtc1dw3Yv5/FASehytcke04H3lncPYryitQajCUSIiV4pIpH1cCZSEc2DK4Us/R+6RQBllv9ls2Q722bOEtITAwRKbw0F7xnHek58x87+rMcYw87+e8GgnP7qIxxaup6au8TS6VbWeOiXlVQ3UVJQji1CF46dYLrI7gR3AxXhcZxXFi3vO8xjbjYFdpZYrryssyYdrd1NbV0/hvkoA6lu4y9y5S724zPsFv/3AIb8ZzRMfbuDlr39otN/q2nr37nnXUpiiKKF7VW01xkw2xmQZY7oaY84H1KtKCUh8TCSPXTrSfX3cgx9y95vfs+eg5+X74pdbWV5oGbS/LzrAtr0VzF1a2KxIwuXVHvtD7/QEr3s/nb2Ei3Oz/dqUNCIEX28qYdm2/W6X5T0Hq9rFxVhROiItyQB4S6uNQul0XJibza/OGOi+9v3VP/O/q92bCjcWlzP+kUXc+u/lvPxN4zMBX5zhTVbvKGVTsSfA4rpdZUTbYVPSEz1LYs44XABb9pRzqMYjQI8uWAdYs4746Egemr+WUTMXhjRTUZTOTkuEo31T2ikdntQAyaOc7Avwq39bM5JAlR3y9qQ67Q8fe127Xvbf3n2mu+yrTZ49GnX1hlN/v5ifPu9JjbulxDKypyXG0C8r0Z058aO13nnXfVm8bjeX/u1Lt61FUTojLREODX+qNEgg4Xj04hEkx0URHSkBl4ve/K6ItTtLm/Q5zhS2vvjuJL/5jAEAXvlCXC/5LzZ6/D1cy2opcdFeBv5g7r4ufvXqMr7ZvJdvbY8xRemMNCgcIlImIqUBjjKs/RyKEpQu8f7CcUleL07sn+HOA/KbcwfxyMUj2PTguSTHRrGrtIqJj3/q9rwKhTJbOC7L807nkpUc696M6OLmMwYyqFsyA7om+bV34Wxzy5kD2epw4f32h4YFwdV2d5l6YSmdlwaFwxiTbIxJCXAkG2NCjXOlHKGkOtxs/+ekvrx383jAexaQlRzLpXm9iIgQujhmKJf+zbNxsDFcG/WuO9k7i2HP1PiA9Yf37OK1vFVe5bFtzF1a6N6/cee5gzljyFE8dOFw9/2q2npyZrzDEx9u4MH5a8iZ8Y6X8Lie7WADsyBFOdxpyVKVojRIqmPGcfd5QxjUzUpfG+cI4d430/PLPzGmeb9FXC/uFJ8ZTs80j3D85Ypc93lGUiwl5VVuN94Kx0a/W/+9nD99YMXvjI+xxomWU8QAACAASURBVHneCP/J9WML17s3Mg6/7313ea09k6qsaXyfiKIcrqhwKGEjmHF8aI8u7vNRvVLd5wmOXB+uPR/GGL89FPsrqlm61bOUVWrPHnztEUkOIcpweFRlJMZQU2cos+0VldXe4UT+Ye849y1vjINVte4+nR5aitLZUOFQwoYr30eyT7rZaSfm8NilI1n127O9yrOSrCi8qQnR7DlYzVl//JgP1+wm9/6FfL3JMlx/vamEUTMXctHTX7rtCaWHaoiJjCAuOpLj+noCKo4f6EmNG+uY5WTYolRiG8Arg7zkzxp6lF9Z/l2B85ftLa/2WrL604cbeOnrlodSUZSOSFiFQ0Qmisg6ESkQkRkB7k8TkWIRWWYf1zru/URENtjHTxzlY0Rkpd3nE3YmQKUDEhkhPHrxCN68cZxf+YW52ST6CEqmHb69b6YVsmT9roOsLDoAWHGunlpUwGXPuLMIu72hSitrSYm3+nrm6jxevu44tsyaxKBunjwjKY5sh679HO9+v4OtJeVUBJlZOG00f7xsJLm9U8lMivWavbjIvX8hJzz0kVfZnW9ockqlcxI2A7eIRAJPAWcChcASEZlnjFntU/VVY8yNPm3TgXuBPCy336V2233A08B1wNdYSaImAu+G6zmUlnGJj6dTQ7jS154/qiff/WDtKnfFlCopr+bZTzd71S+trKFLfDSfF+xxe3ClxEVzYn9rptEl3vOCd8bDyrRnNo+8t45H3lvnFVvLiXOmdMHobC4Ybe1AdxrWJ4/swbzl20N+RkXpDIRzxjEWKLCTPlUDc4ApIbY9G1hojNlri8VCYKKdvjbFGPOVsSybL2LlHVc6Ab84bQA/Pq43l+Rlu0OWfL/d2tPhDFfiYmPxQapr6/lhbwWDuqf43Xe6AzsN566lKhebisv92vboEhc0na7TK+zkRnJ/1IYQTFFRDjfCKRw9AWcmnEK7zJeLRGSFiMwVEdfP02Bte9rnjfWpHIZ0SYjmwQuGkxATxdlDuwGwzt4MWFxW5edeu+dgtXtDXl6fNL/+YqI8/3k7X/bpAZaafLn//GFB79113mB6psaz+NZTuXiMdxysbilxTBre3X3tCquiKJ2J9jaOvw3kGGNGYM0qXmitjkVkuojki0h+cXHrpRNV2obE2ChioiLcM401O0rpb2/au/u8IQAcqKxxLxslxYa+6hobFUmUz2wiQiyjvYt+WUkE44rj+vD5jNPIsW0x3959Jmm2B1mP1DieuiKX319izZg27fGfzSjK4U44haMIcC5wZ9tlbowxJcYY1xbbvwNjGmlbZJ8H7dPR9zPGmDxjTF5WluacOhwRvHdxL9+2n6O7Jrlf8KWVNZRVWZ5MLg+uUPnunjMZ45il/OHSkdw3eSgf3HIKPz+lP318ouw2RHpiDFcc1weA6Ejrf6nTB3UFPDMmRelMhFM4lgADRKSviMQAl2Oln3Vj2yxcTAbW2OcLgLNEJE1E0oCzgAXGmB1AqYgcb3tTXQ28FcZnUNoRZyIlsGYYURHiXnb604cbOFDhEo7AM47TBnVl0ojufuXJcdHMnOLJG+KauRzdNYkZ5wwKat8Ihiucu8vTKy0xhvTEGB6cv9Zrg2FHoWD3QR55b61XLhNFCZWweVUZY2pF5EYsEYgEnjPGrBKRmUC+MWYecJOITAZqgb3YyaGMMXtF5H4s8QGYaYxx7fi6AXgeiMfyplKPqiOIov2VXtcL7XzlwYTjuWnHBu3LtZMdoFcTZhiB6NbFysfunCG5Ni4++VEBt08c1KL+W5trX1jClpIKrjqhD927BA7NoijBCGu8KWPMfCyXWWfZPY7zO4A7grR9DnguQHk+ENxyqXQa5t04jsl//tyrzDeEuivHeFNsHC6cBvMJx3Rtxgg9uPaMXH1CjrssJyOBLSUV7OyABnJX8qu95dUqHEqT0UCFSodlRHZqo3UWr7McH5pq43Dxlyty/ZI6NYeuKXFseOAcL6P7iz89jpMfXUSvtI73Yo62x/nt1n1eIWDAciEu2l9Jn4zA+1sUpb29qhQlJO6aNNjr2pmaFoIvVTXGucO7c9og/9AizSE6MgJnIIPeGQnERkX42WpKD9Xw27dXcaimjoffW8uE3y9ulc8PlVXbD7DdngUt2eIfJv4PC9dzyqOLKWxGUi3lyECFQ+nQvHPTSfz96jyuHd+Pdb+byLrfTQSs1LRjczxxqZwRdzsSCTGRfLdtPzkz3nHH23rqowJmf76Ffy8t5OnFG9m8p7xNjdSTnvjMfT5v+XaqfYTNNc6OuMSmdAxUOJQOzdAeXThjiDUjiI2KJDbKIxC7y6wXW6/0jrcU5CIhJsqdlMoVZ8uVxKrKEVyxvImReJtLIIG6+03vmFqu79h3pqQoLlQ4lMOe+6d0XF8JXy8wgKhIazmr1uGBtbQJqWYfmr+GsQ980Kzx/PXjTX5lq3Yc8Lp27bivqtXQ8EpgVDiUw5b/nXA0AHmOJauOzoHKGrc3lzPfx0+e+4Zh9y7wS3UbiL99sondZVVNXt6qqrVsKi5cM7VDPkmnYm3hOFilwqEERoVDOWy5JK8XW2ZNapYrbnvx0Pw1bs+rRet2e907WFVLycGGc5U7E0QFCwcfDFeKXYDbJw5i/AArokLB7oNe9Vz2ogMV/oElFQVUOBQlrHz66wle13sOVvPRWkswVhRaS0TOoItPf7yxwWWrwn2epS/fzIjBcC05Haj0JJq6/tT+fp5qNXX1VFbXuZfSNgaIGqwooMKhKGGlV3oCX8w4jS/vOA2A2OgIVm33jl/V27FrffbnW7jo6S+C9rfHMSPZF8KM4O3l2znmrvfYvKecA5Xe9RNiohh3dIY7/PyAO99l8D3vsW2v5Ya7bmdZo/0rRyYqHIoSZnqkxrt3Z7+zYofffd9w8QAFuwO/tJ25Q/aWV3Oopo6lW/dSH8A2UltXz3urdgLw/qqd3PPWKr86ORmJREWI17LZ1hJLOAqKD/rVVxRQ4VCUdsc3sRTAbXNXBKxb40gM9fwXW/jVq8u46Okvueq5r/3qXvWPb9xC9d0P+9051j+45WR3nYSYSErKq7lm9hJ32e4ya1ZTXFbFh2t2cfeb31NVW6cBERU3h49VUVE6IT1T4+3QKlsD3q+rN/ywt8Kdh90pHFU19e5YXZ8XlPi1/XKTp8w188hKjuXorp5c7PExDb8C/ueFfAD++dVWuibH8s2dZ4TwVEpnR2ccitJG3HzGAK/rLbMm8fmM0+iabOVAj3fsfnflXH9kgRWSxLUfpNoWjv5ZiSTERHJcX48r8uYQkkY50+mCNeMIxEW52X5lu8uq3GHjOxI1mp63zVHhUJQ2IssWCF/GD8hk9rRjeeemk0j2cS3+aI1le9hvG8Jraq3lom5d4igpr/bagzHh94sxxrDnYBXrdwW2kfT2CR/vFI6ju3qyHvbLSiQzwBLasHsXBH2+9mDBqp0MuPPdoM+rhAddqlKUNiIzySMcj148wn0uIkywMwau/O3Z3Pjyt6y2Pa822HssXOE/aurqiYwQMhJj+bxgu3u24uIvizfy6IJ1QccQHemdoMo5y6mqrSM5LoqyQ7Ukx0W50/Z2ZP5r23C+LzrAwKOSG6mttBY641CUNqJbSpz7/JK8XkHrJcZEUe6TNfCgnYekpr6e6Ejh7RXbAfisYA9DunsSUvmKRnSkeM0yrh3fz+v+ucM92RG37a0kwo7um9JAmPraDrA0tOdgFc99ttm9+15dh9uWsAqHiEwUkXUiUiAiMxqod5GIGBHJs6+vEJFljqNeREbZ9xbbfbrutSwDj6K0EUN7pDReCYiPifTbFe6yLZQcrCYlLtqdE72iuo7VO0q57exjAva14YFzGdbT+ty/XJHLsT7hWRJjo5h14XD3tWuTYEp88MWIJz8qCOk5wsmM11cy87+r+cDOAPm3T/xjcCnhI2zCISKRwFPAOcAQYKqIDAlQLxn4JeD2JzTGvGSMGWWMGQVcBWw2xixzNLvCdd8YsxtFOQyIiozg27vP5L2bxzdYLzHWEo6lW/e6y2546Vsu+esXzF1aSE5mIn//iXdK3JR4/xnCwKMsm8UvThvAqF6pjO4dODHWpfbsZ9qJOe6ymMhIvrnzdK46vg/rf3cOW2ZNcuc86Qi/7l2C4aJfliadakvCOeMYCxQYYzYZY6qBOcCUAPXuBx4GggX/n2q3VZTDnvTEGK9c54FIiImirt5w2d++8ip3JV36ZvNev1DyvkZ1sBJLAQzunsKb/zsuaIrYiAhhy6xJ3Dd5KJNH9rDbCl2T47j//GHuaLnzbjwJgKNSAhv52xPfLIZKeAmncPQEtjmuC+0yNyKSC/QyxrzTQD+XAa/4lM22l6nuFmfKNe++p4tIvojkFxcXN2P4itI+uGYPwbbbXTe+L7FRkV6xprIDpKcd1oyX6f3nD+Pu84Ywtq9/xOG+mYn0yUjghS+38vgH65vcd2vidDQAOHioJkhNJRy0m1eViEQAjwHTGqhzHFBhjHFmmrnCGFNkL3G9jrWU9aJvW2PMM8AzAHl5ebrlVTlscBnRg4VYv+3sQYBl6O6RGk+P1HhG9UolMkKoqzc8fUUuqQkxQZemGqJLfDT/c1LfoPeH9ezC1pIKHv9gAzefMbDJ/bcWdfXeBvqOuL+kMxPOGUcR4HQdybbLXCQDw4DFIrIFOB6Y5zKQ21yOz2zDGFNk/y0DXsZaElOUTsOJ/TPc5/2yEln4q5O97ruWjsDyihrVyxKI56+x7B65fdI4oX9GWNLpPnC+J2lWe268q603nDIwi7w+aQzrmULZIRWOtiScwrEEGCAifUUkBksE5rluGmMOGGMyjTE5xpgc4CtgsjEmH9wzkktx2DdEJEpEMu3zaOA8wDvvpaIc5iQ67BWbissZcFQymx86t9F24wdksWXWJI5yuP22Nl3io935Twbc+W7YPqcxausMA49KYu71J3J0VhL7K3Spqi0Jm3AYY2qBG4EFwBrgNWPMKhGZKSKTQ+jiZGCbMcbpZxcLLBCRFcAyrBnMs608dEXpcAQx5bU5IsLDF3k2LzoTS7UldfWGyAjr9bVkyz52lh7i+6IDjbRSWouw2jiMMfOB+T5l9wSpe6rP9WKs5StnWTkwplUHqSgdmJtOO9p9PjK7C9lpCQ3Ubhu6dfEYpj9ZX8xZQ7u1+Rhq6+vdmRRdcby+2lTCsJ4NOwQ8taiARxesY/m9Z/nF7VJCR3eOK0oH5Bg7fMYNEzzC8daNJ/HUFbntNSQ3WUmepbDp/1za5p9fX2+oN7gzFY7MtsSiPoSw766d9TfP+S58AzwCUOFQlA7IS9cdx7NX54XFwN1SemckcMZgT8CGYN5f4cI1wxAs4XjpOmth4sH5a9m+vzJoOycdYRPj4YwKh6J0QDKTYjlzyFHtPYygPHu1x/nx10GSToWLG176FoCte60w8kkOZ4L5K/0zLDpx7XcZ0iOF0/6wmILdmuWwOahwKIrSZJzG+te/LWzTz169w4oc7JpxOCmtbNi7yuW2+8Ga3WwqLuf6fy3lP208/s6ACoeiKIcVrmi/lx3r2Sa2/N6zANhb0XAo+HKfjYIbdh/klteWd4iIv4cTKhyKojSLO84Z5D5vi53bRfsr2b6/EmMMPxrZwyssSpf4aPplJbKvPPiMo7q2ntog9pilW/e1+ng7MyociqI0iwtGe0LP/XT2Egp2t9zgbIzh7eXbA+4PGTfrI06c9RHl1XUkxfo7DWQmxlJcVhW078rq4HtOSnXneZNQ4VAUpVl0TYnjk9smAPDNlr2c8dgn7DwQLMh1aHy5sYRfvPJdg1kMi8uqiI/234LWv2si32zZy6J1gTMtVNQEF4cNrSB6obJq+wHe+34n4Ml/crihwqEoSrNx5ehwMXfptiA1Q8P1y39rSXmD9Zzxuly4Qq1cM3tJwBmLKyx9INrSu2rSE5/x838t5bsf9jHyt+9z7QtLDrtd7yociqI0m0SfPCA7WjjjcAWTb2xvSCBhcOVlBzj9Dx/73b/pFWvT3+OXjeKuSYN57NKR7nuB8qvvPHCIgXe9y7a9FaENvYn8wh7PB2t2c96Tn4XlM8KFCoeiKM3G95d/MDtCVW3jMa3q6w27Si0bRSAjtjN3uq93FIBz43hRAxsBRayQ9BeM7slb/zuO8QMyKQuQz+Oip7+gurae8Y8sCvh5LaVwn/cYd5U2XXQrq+t4cP6aNg8rr8KhKEqrMPCoJL9c6QDfFx3gmLve4y+LCxg1832WbdsfsP2tc5dz77xVAOwL4FZbU1dPvL2TPtDnHNfPO/mU82U66G5PJN+TB2QB1l6Ukb1SSY6LChiWffyATPf504s3Bhxza7J+V9PsLGWHahh8z3s888km/u+1ZY03aEVUOBRFaRXSEmLYEsA2sdJev3/kvXXsr6jhrz4v4W17K6iqreM/33rS9QQKk15dW8+EQVmkJkRz7Xj/ZFMTjunKd3efyeX2/g6n3eJQjbWMNaBrEmmJMV7tkmOjA844spI9wRz/vKiA6trw7PW4NC8boMn9X/6MJ7XwglW7ePHLLa04qoZR4VAUpUV8+usJvPjTsQzunsLaADGgXsv3Npi/t2qn+/z9VTsZ/8gi/vnlVq86hfsq/ZaHqmvrOSoljmX3nMXo3mkBx5KWGMPPTukPwKZif4P3IxeP8CsLNuMor/Ke1SxwjLs5VNfWc/yDH7qvoyOFpXedwfST+wEN74UxxvDGd4XMW76dAxU1vL9qp5/t5Z63VgUUwHDQbqljFUXpHPRKT6BXegLv2i6mBbvLOLprModq6rj2hXy++8F/aaq+3rCx+KA7um6g5auh9y7g9etPIDYqkt4ZCVTX1RMT2fhv3XR7RrG33H+5KyrCv31yXDQV1XXU1NUT7ei/orqWrsmx3Dd5KDe89C2VLcw9sr+imp0OO8bi2yaQkRRLtb1rvbyqjh0HKikuq+K5zzbz2ynDmPrMV4wfmElORiJ3/GdlwH7H9Elzb2Cc8fpKbpjQn6O7JhEbFb4AmWGdcYjIRBFZJyIFIjKjgXoXiYhxpY0VkRwRqRSRZfbxV0fdMSKy0u7zCekoGW4U5QjHZRM447FPOFBRw/pdZXxWsMerzq8nHgNY+xfO/OMnAfvpn5XoPr/o6S8578nP+Pk/l1rCEcAN15dk29Or9FAtNXX1rNrucXWNiw4kHFb9gz6zjorqOhJjoxh3tPVcjcXBaoxqR1iTP/94ND1TrYCLLs+0bfsqOOGhj5j85895c9l27v/valbvKOVvH28KOtsZ1C2ZJ6eOdl8vWLWTSU98xu1hDjwZNuEQkUjgKeAcYAgwVUSGBKiXDPwS+Nrn1kZjzCj7+Lmj/GngOmCAfUwMx/jblb2b4NnTYck/oFJDISiHB5lJHpvAe6t24OsYddvZx9A12dprMfr+hV73nN5YCTFRbiO4iy82lhBCug0AIiKEhJhIKqpqufof3zDpCcvVNbd3KgPsPCdOXMIx+v6F7C47hDGGDbvKmLd8O8YYkmOjEGl8s97LX//A1pJyHn5vLec9+anffacLcUKM5/kS7Gddvb3Uq/7cpZ7gi4vXFQf8zKTYKLp38eRHcXmjLVi1q8GxtpRwzjjGAgXGmE3GmGqs3OFTAtS7H3gYaNQXTUS6AynGmK+MMQZ4ETi/FcfcMTi4G6rL4Z1b4PfHwL+vgYIPoL590nQqSig4f83f/vpKLzG4ccLR/O+Eo0kNknXvw7We3d4p8VH869qxPDctj0vGZHvV27ArtI16ibFRlFfX8uWmEndZIPsGWEtVLv728SbmLNnmng31SI0nIkJIiYtuUDg27ynnN2+sZMpTn/P04o18X1TqV8dloAe88qxE2ctjH68PLA4Nkb91HyLCT8d5Owu0dFmtMcIpHD0Bp1Ws0C5zIyK5QC9jzDsB2vcVke9E5GMRGe/o0xkD2a/PTkHv4+GGL2H6YhjzE9i0CP51EfxxKHxwH+zZ0M4DVBR/nC/DxJhIKh0hPi6yBaBLgrdwZPh4OI0fkMnvLxnJmD7pnDboKB69ZCRDe6S479eEGMU2MSaSLXu8jcdHd/WfbYBlpHbx6YZiL7fY8+14XKkJDQvHikLLRuP0Bquo9l76cr7Mk2Kbbl4ekR08Le5dkwb7zdJ8P781aTevKhGJAB4D/i/A7R1Ab2PMaOAW4GURSQlQr6H+p4tIvojkFxc3XcnbHRHoMRrOfRT+bx1c8gJ0GwGf/wn+nAd/PxPyZ8OhwytUgdJ5cebwLq+u89pr4XpRZjmWswD+eNkofuHIq/6784fRvUu8Vx2XwTojMYbfXTAspLEkxkZ5zTZCZf2ug8z+fIv72iVsFdV1vLVsu9tl1hjDm98Vsav0ED+UVAT0iBr52/e54SVPal3nUlW6j2CGwuBu/q/An9keWRER4jfLCOdu9HB6VRUBvRzX2XaZi2RgGLDYtm93A+aJyGRjTD5QBWCMWSoiG4GBdvvsBvp0Y4x5BngGIC8vr21zW7Y2UbEw9HzrKNsJK16FZS/Df2+G92bAoPNg9BXQ9xSI6HipRpUjg67JsSTHRlFmv0Qf/8AzM3YJR+/0BDKTYtwhPvpkJHi9UH1DmIDH4+rSY3v5iUowEmNCf7WdMjAr6D3Xng9X1N0vN5VwysAs1u4s4+ZXlzG2bzrfbN4bsG1NnWH+So9R27V099vJQ8lOS/Cqe/vEQTz83toGx3l8/3Qqaup4e/l2d9k14/z3s7jYVNxwvK+WEM4ZxxJggIj0FZEY4HJgnuumMeaAMSbTGJNjjMkBvgImG2PyRSTLNq4jIv2wjOCbjDE7gFIROd72proaeCuMz9DxSO4G434JN3wF130Eo6+EgoXwzwvg8eHw4UzYU9Deo1SOQESElb89mwnHWC9i1wa8Ub1S3faPiAgh/64z3W36ZCTSO8PzEk1P8P8lPryntUSTk5Hgdy8Ya3b42xiCEdWAi2+3lDiva1fCp+ftWUkw0RjdO9V9PvCud6moruWQPVtxeWkFqw9w5pCjuH/KUO6fMpRPfz2BL2acxgWjs3ly6mjy+lh7WB65eATdHIbxj287FYArj+8d9Hlai7AJhzGmFrgRWACsAV4zxqwSkZkiMrmR5icDK0RkGTAX+LkxxvUvdAPwd6AA2Ai8G7iLTo4I9BwDk/4A/7ceLp4NXYfAZ3+EP4+Bf5wNS1+AQ6H/D6QorUGvdO8X/MvXHeeVataXAV2TGZHdhZ+f0p+ICP96824cx/ybxnNpXq8ArQPjmvW8Ov34kNsEooftMjvvxnEAfGUvf+0qC+7L0yU+2h3WBKyNf9v2VnLADqMSyL7h+53lZCRw1Qk5XHVCDr3SE9zjAIi0vyPffvpkJLJl1iRyg2yObE3CugHQGDMfmO9Tdk+Quqc6zl8HXg9SLx9riUtxER0Hwy60jtId9lLWS/D2TfDu7TD4RzDqx/ZSlgYLUMLL7RMH8aJjJ3hcgI1opx6T5d4ZHhkhzLvxpKD9iQhDejTJxEn3LnHsOHCIsX3TyUyK4eiuSQ3W/9f/HMeu0kM8OH8NJQE2Dg6wDevPfrqZkwZkERVA4FxUVNf6B3+sqeOHvRXERkVwVEqsXxvXzObOcwczJieNkdmpfnVcZNqhUIKFKOmX5XnW+noTUIxbiu4c72ykdIeTbraWs4qWWgKy8nVY+RqkZMOoqTByKmT0b++RKp2UxNgo4qIj3O6ngV5cz18zNqxjeOOGcRTsPoiI99JYME6yNy/2Sk/grx9vZFiPFLIds4D4mEi6Jseyu6yKnzz3DSf2zwjYT9fkWG47+xi/cCn7Kqopr64jOS4q4OwrMkLYMmtSSM92xuCuvLNih9f+DSejeqVy4eie/Oe7Ig5U1vjF5moNVDg6KyKQnWcdZz8Ia9+xDOqf/B4+eRR6nwCjrrAM7rGB3RQVpbl8fccZjJz5frt9frcucV7r/6Eytm+6Vy5zJ8f1y3Abpr/YGNhj65s7zwDgQEUN81fu5Jst1gr7/opqKqpqSWiC0T4YF4zOZkzvdC/bkC/jB2byn++KmPTEp3x066lertKtga5bHAlEx8Pwi+Gq/8CvVsHp90J5Mcy7EX4/EP7zM9j8CdSHJ/qncuTRJSGa68b3ZWSv4EsuhxsJPi/fhoIddUmI5tWfeewrv3p1OW8u2+61Y7wlNCQaAKm2k8H2A4daXTRAZxxHHl16wvhb4KRfQeESaynr+//AijnQpbe1lNX/NEjtA0lHqU1EaTZ3TvKLMHRYExnprRSnD+rKj4/rzbE56Qy/z392FWhJKlyh2X0JtkO/tVDhOFIRgV5jrePsh+ylrJfg40fg44etOpGxkNrLEpHU3pBm/03Nsf4mZjb8s0tROhEpcd4v49ioSE4bdBQAf7hkpNcGyGBs2hO+vRVOXAEUp52YE5b+VTgUiEmAEZdYR+l22LUK9m+FfVth/w/W+Y5lUOGzrhudYAuJr7DYf+PTVFiUToMr9ElGYgwl5dXEO5adLvKJqeVi7f0TGXT3e20yPiddU+L47y9OanRJq7mocCjepPSwjkBUlcH+bZaQ7P/BFhb7+OErqPIJfxKb0rCwxDXNxVJR2pPzRnSnR2o8n6wv5k8fbggp3pSvfWFI97b7b35Yz+CxrVqKmFBjFR/G5OXlmfz8/PYeRuencr+PqPzgfV3jM02PT/MWkoz+0PtEyDpGZypKh6WiupZ/5xdy5fF93JvxGuJQTR2D7n6P/53Qn1+cNiAsxupwISJLjTF5fuUqHEqbYAxU7IX9WwILy/4foNbejZvYFfqOh74nQ854SO+nQqIo7UAw4dClKqVtEIHEDOvoOcb/fn29JSpbPrNcgzd/Ct/bwQNSenpEpO94a3aiKEq7ocKhdAwiIqyZRXo/yL3amqGUFMDmjy0R2fA+LH/FqpuWY4vIKZaQJHdr16ErypGGCofSMRGBzAHWcey11oykeI1nNrJmHnz3T6tu5kBbSOxZjUXNmwAADspJREFUSWLgcBCKorQOauNQDk/q62DnCktENn8CP3wJ1XZa0a5DLRHpOx76jIP4zrN7WVHaEjWOq3B0bupqYPt3lohs+dRyD649BBJhZU7say9t9T5eY3MpSoiocKhwHFnUVkFhvkdICpdAXTVIpGWcd3lt9TrOiuWlKIofKhwqHEc21RWw7WtLRDZ/AkXfgqmDyBjIPtZKgpXSA7pkW15cXXpCcg+Iav2Q1IpyuNAu7rgiMhH4ExAJ/N0YMytIvYuwMv0da6eOPROYBcQA1cBtxpiP7LqLge5Apd38LGPM7nA+h9IJiEmA/hOsA6xd8D98ZXltbfncyldy6IB/u8Suloik9PQISkpPW2B6QHJ3iAxvQDlF6WiETTjsnOFPAWcChcASEZlnjFntUy8Z+CXwtaN4D/AjY8x2ERmGlX62p+P+FXYmQEVpHrHJMOBM63BRddCK1VVaCAeKoNQ+DhRZrsGbPobqMu9+JMKKIpzS03/G4hKb5G4QcfjsFlaUxgjnjGMsUGCM2QQgInOAKcBqn3r3Aw8Dt7kKjDHfOe6vAuJFJNYYUxXG8SpHOrFJkDXQOoJxqNQjJqWFltC4znevgYIPoKbCu41EWjOTlB7+M5aEDDD1UF9ruRzX11pLaPW1LSyrsw9nWb2nvqm3ohu7ha6XNbakbhCpXvqtRl0NlO2AiGjrB0QniYAQzv9CegLbHNeFwHHOCiKSC/QyxrwjIrcRmIuAb31EY7aI1GHlJf+dCWCoEZHpwHSA3r11p7HSSsSlWEfXwYHvGwOH9vvPWEqL4EAh7FgB6971hFdpbSKirEMi7fMInzL7QKxkXlWl3u1dItcl2yFytqh0ybbSDyekd5oXYIuor4eDuzz/ts4fFAfssoO7APv1FJ1gb3LtC+n9rfMM+29y98PqO223nxYiEgE8BkxroM5QrNnIWY7iK4wxRfYS1+vAVcCLvm2NMc8Az4BlHG+9kStKA4hYwRvj06DbsMB1XHG7Sougcq/Pi971cm+szFcQopqXdMs9gyr0HK7rom9hzduWN5qTqHhvIXGfO0QmJrHpY+lIuP+NCr2F33e2We+dW5zoBM9S5dFneAS4rhpKNsHeTVC8DtYv8P5eo+I9ouISk3SHqHSwhGrhFI4ioJfjOtsuc5EMDAMW25myugHzRGSybSDPBt4ArjbGbHQ1MsYU2X/LRORlrCUxP+FQlA6LM25Xe9PYDKq+Hir2+IuK63zjh1C2E/evahfxabaoBJi5JGQ4BDHKcfiKYzPFMBR8lxwDiUNtpXebiGiPHavX8f7PldIz9Bw09XXWZ+3dBHs3wt7NULIR9mywwuv4iUpfT0gep7C0k6iEzR1XRKKA9cDpWIKxBPixMWZVkPqLgVtt0UgFPgZ+a4z5j0+fqcaYPSISDbwCfGCM+WtDY1F3XEUJI3U1tlOB/cI9sM0hMPb1of3N7Fx8RMZHbCQygABFBBYkY6ylowNF/rljJMKy73jZoHp6z6wSs9rmJV1fZ31/JRttYbGPko2wb3PDouIUl+QeLR5vm7vjGmNqReRGLI+oSOA5Y8wqEZkJ5Btj5jXQ/EbgaOAeEbnHLjsLKAcW2KIRCXwAPBuuZ1AUJQQio60kXWl9gtepLveISOU+h0NArb8x31nm5QAQ4H4obepqoKbSEo60vpBzkr84dCS36ohIO09Nb4/7uAuXqLiExC0qBbBhIdQ5TMFRcdbzXvZPK+ZbK6IbABVFUToD9fW2qDgFZRNM+bPl0NAMNB+HoihKZyYiAlJ7WUe/U8P7UWHtXVEURel0qHAoiqIoTUKFQ1EURWkSKhyKoihKk1DhUBRFUZqECoeiKIrSJFQ4FEVRlCahwqEoiqI0iSNi57iIFANbm9k8EyuxVEdFx9cydHwtQ8fXMjr6+PoYY7J8C48I4WgJIpIfaMt9R0HH1zJ0fC1Dx9cyOvr4gqFLVYqiKEqTUOFQFEVRmoQKR+M8094DaAQdX8vQ8bUMHV/L6OjjC4jaOBRFUZQmoTMORVEUpUmocCiKoihNQoWjAURkooisE5ECEZnRDp/fS0QWichqEVklIr+0y+8TkSIRWWYf5zra3GGPd52InN1G49wiIivtseTbZekislBENth/0+xyEZEn7DGuEJHcMI7rGMd3tExESkXk5vb+/kTkORHZLSLfO8qa/H2JyE/s+htE5CdhHt+jIrLWHsMbIpJql+eISKXju/yro80Y+7+LAvsZJIzja/K/abj+/w4yvlcdY9siIsvs8jb//loFY4weAQ6snOYbgX5ADLAcGNLGY+gO5NrnycB6YAhwH3BrgPpD7HHGAn3t8Ue2wTi3AJk+ZY8AM+zzGcDD9vm5wLuAAMcDX7fhv+dOoE97f3/AyUAu8H1zvy8gHdhk/02zz9PCOL6zgCj7/GHH+HKc9Xz6+cYes9jPcE4Yx9ekf9Nw/v8daHw+9/8A3NNe319rHDrjCM5YoMAYs8kYUw3MAaa05QCMMTuMMd/a52XAGqBnA02mAHOMMVXGmM1AAdZztAdT4P/bO/+Yq+o6jr/e09TEEDRiTPuBiPmH6UOSmqFTYihlTlMT54aZW1naVmzmirLa+sG0WVuy3FREGjFjoZKzAeKKxgARhAcy80drFREoMjCxwsd3f3w/Fw753AcO4z738vh5bWfn3M8959z393Oeez/n+/2e5/Phgdh+ALisYp/twgpgiKQR/aDn48CLtvvKINAv/rO9FHill8+u46+LgMW2X7G9DVgMXNwqfbYX2X4jXq4ATuzrHKFxsO0VLr+CsyttOuj6+qDZNW3Z97svfdFr+Awwt69ztNJ/B4MMHM05Afhb5fXf6ftHu6VI+gAwBlgZpptj2GBmY1iD9mk2sEjSakmfD9tw25ti+5/A8DZrnMzeX9ZO8h/U91c7tX6OcgfcYKSkpyX9TtJ5YTshNPWnvjrXtF3+Ow/YbPv5iq1T/LffZOA4BJB0DPAr4Cu2dwA/A0YBXcAmSte3nYyz/WFgEnCTpPOrb8YdU9ue+5Z0BHApMC9Mnea/vWi3v/pC0jTgDWBOmDYB77M9BpgK/ELS4DZI6+hrWuEa9r6B6RT/1SIDR3M2Au+tvD4xbP2KpHdQgsYc2/MBbG+23WP7TeAe9gyntEWz7Y2x3gI8FHo2N4agYr2ljRonAWtsbw6dHeW/oK6/+l2rpM8ClwDXRnAjhoC2xvZqyrzBKaGlOpzVUn0HcE3b4b/DgU8DD1Z0d4T/6pKBozmrgNGSRsYd62RgQX8KiPHQ+4A/2r6zYq/OCVwONJ7eWABMlnSkpJHAaMoEWys1DpL0rsY2ZRJ1Q2hpPOlzHfBIReOUeFroHGB7ZYimVex1l9dJ/qtQ118LgYmShsawzMSwtQRJFwNfAy61vbNiHybpsNg+ieKzP4fGHZLOib/jKZU2tUJf3Wvaju/3BOBZ27uHoDrFf7Vp9+x8Jy+UJ1qeo9wFTGvD54+jDFl0A2tj+QTwc2B92BcAIyrHTAu9f6IfnsKgPJWyLpY/NPwEHA8sAZ4HHgeOC7uAGaFxPTC2xfoGAVuBYyu2tvqPEsQ2AbsoY9c3HIi/KHMNL8RyfYv1vUCZE2j8Hd4d+14R130tsAb4VOU8Yyk/4C8CdxGZKlqkr/Y1bdX3uzd9YZ8F3Ph/+/a7/w7GkilHkiRJklrkUFWSJElSiwwcSZIkSS0ycCRJkiS1yMCRJEmS1CIDR5IkSVKLDBzJgEBST2QXXSdpjaRz97H/EElf2o/z/lbS2IOn9NBH0ixJV7ZbR9I+MnAkA4XXbXfZPgP4OvDDfew/BNhn4GgX8V/GSdKRZOBIBiKDgW1Q8nxJWhK9kPWSGhlQpwOjopdyR+x7a+yzTtL0yvmukvSkpOcaSegkHaZSo2JVJNb7QthHSFoa591QSVq3G5V6DLfHZz0p6eSwz5J0t6SVwO0qNToejvOvkHR6pU33x/Hdkq4I+0RJy6Ot8yLHGZKmq9R06Zb0o7BdFfrWSVq6jzZJ0l0qtSseB95zMC9WcuiRdzXJQOGdKsVxjqLUMRkf9n8Dl9veIendwApJCyg1L06z3QUgaRIlrfbZtndKOq5y7sNtn6VSHOjblNQRN1DSf3xE0pHAMkmLKLmIFtr+fqSSOLqJ3u22PyRpCvATSg4oKDmJzrXdI+mnwNO2L5M0npJauwv4VuP40D402vZNYILt1yTdCkyVNIOSguNU21YUYAJuAy6yvbFia9amMcAHKbUthgPPADP366okA5IMHMlA4fVKEPgoMFvSaZSUHT9Qydj7JiU19fBejp8A3O/Iw2S7Wk9hfqxXUwrvQMkNdXplrP9YSp6hVcBMleSUD9te20Tv3Mr6xxX7PNs9sT2OkpIC209IOl4lc+oESm4l4r1tki6h/LAvK6mNOAJYDmynBM/7JD0KPBqHLQNmSfplpX3N2nQ+MDd0/UPSE03alLxNyMCRDDhsL4878GGUfETDgDNt75L0F0qvpA7/iXUPe74zAr5s+y2JBSNIfZLyw3yn7dm9yWyy/VpNbbs/llLY6Zpe9JxFKWR1JXAzMN72jZLODp2rJZ3ZrE2qlGFNEsg5jmQAIulUSmnQrZS75i0RNC6klI4FeJVSjrfBYuB6SUfHOapDVb2xEPhi9CyQdIpKpuD3Uwr13APcSykh2htXV9bLm+zze+DaOP8FwMsu9VgWAzdV2juUUpXvY5X5kkGh6RhKgsfHgK8CZ8T7o2yvtH0b8BIlxXivbQKWAlfHHMgI4MJ9+CYZ4GSPIxkoNOY4oNw5XxfzBHOAX0taDzwFPAtge6ukZZI2AL+xfYukLuApSf8FHgO+0cfn3UsZtlqjMjb0EqW05wXALZJ2Af+ipMPujaGSuim9mbf0EoLvUIa9uoGd7Em7/j1gRmjvAb5re75KvYy5MT8BZc7jVeARSUeFX6bGe3dIGh22JZTsxt1N2vQQZc7oGeCvNA90yduEzI6bJP1MDJeNtf1yu7UkyYGQQ1VJkiRJLbLHkSRJktQiexxJkiRJLTJwJEmSJLXIwJEkSZLUIgNHkiRJUosMHEmSJEkt/gdIqwzPbeQP1AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.recorder.plot_losses()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.491651</td>\n",
       "      <td>0.427178</td>\n",
       "      <td>0.807617</td>\n",
       "      <td>00:47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.494276</td>\n",
       "      <td>0.432113</td>\n",
       "      <td>0.804334</td>\n",
       "      <td>00:43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.473254</td>\n",
       "      <td>0.433276</td>\n",
       "      <td>0.806960</td>\n",
       "      <td>00:45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.472309</td>\n",
       "      <td>0.427583</td>\n",
       "      <td>0.809586</td>\n",
       "      <td>00:47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.451277</td>\n",
       "      <td>0.422035</td>\n",
       "      <td>0.807617</td>\n",
       "      <td>00:47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>0.449352</td>\n",
       "      <td>0.425011</td>\n",
       "      <td>0.809586</td>\n",
       "      <td>00:45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>0.421911</td>\n",
       "      <td>0.425715</td>\n",
       "      <td>0.811556</td>\n",
       "      <td>00:47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>0.445819</td>\n",
       "      <td>0.420823</td>\n",
       "      <td>0.814839</td>\n",
       "      <td>00:45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>0.416064</td>\n",
       "      <td>0.420648</td>\n",
       "      <td>0.818779</td>\n",
       "      <td>00:48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>0.410224</td>\n",
       "      <td>0.421482</td>\n",
       "      <td>0.815496</td>\n",
       "      <td>00:50</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Better model found at epoch 0 with accuracy value: 0.8076165318489075.\n",
      "Better model found at epoch 3 with accuracy value: 0.8095863461494446.\n",
      "Better model found at epoch 6 with accuracy value: 0.8115561604499817.\n",
      "Better model found at epoch 7 with accuracy value: 0.8148391246795654.\n",
      "Better model found at epoch 8 with accuracy value: 0.8187787532806396.\n"
     ]
    }
   ],
   "source": [
    "learn.fit_one_cycle(10, slice(2e-3/100, 2e-3), wd = 0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RNNLearner(data=TextClasDataBunch;\n",
       "\n",
       "Train: LabelList (6090 items)\n",
       "x: TextList\n",
       "xxbos in xxup both ' xxunk and times of national emergency . ',xxbos xxunk ok i was n't completely xxunk i may have also been in a food xxunk bc of the xxunk / xxunk / xxunk i also annihilated w / xxunk,xxbos xxmaj ladies here 's how to recover from a # date you totally xxup bombed ... according to men http : / / t.co / xxunk http : / / t.co / xxunk,xxbos xxmaj the xxmaj latest : xxmaj more homes razed by xxmaj northern xxmaj california wildfire : xxmaj the latest on wildfires burning in xxmaj california andû _ http : / / t.co / xxunk,xxbos xxmaj it hurts for me to eat cause i burned my xxunk with a xxunk yesterday !\n",
       "y: CategoryList\n",
       "1,0,0,1,0\n",
       "Path: .;\n",
       "\n",
       "Valid: LabelList (1523 items)\n",
       "x: TextList\n",
       "xxbos xxmaj students at xxmaj xxunk remember xxmaj australian casualties at xxmaj xxunk xxmaj xxunk xxmaj gallipoli \n",
       "  http : / / t.co / xxunk via xxunk,xxbos xxmaj israel wrecked my home . xxmaj now it wants my land . \n",
       "  https : / / t.co / xxunk,xxbos ' xxunk : xxmaj haha jam xxunk xxunk garden city xxunk bumper to bumper with xxup xxunk xxunk decide to chill via xxunk ' xxunk siren xxunk,xxbos i feel like i 'm drowning inside my own body ! !,xxbos xxmaj police xxmaj officer xxmaj wounded xxmaj suspect xxmaj dead xxmaj after xxmaj exchanging xxmaj shots : xxmaj richmond police officer wounded suspect killed a ... http : / / t.co / xxunk\n",
       "y: CategoryList\n",
       "1,1,0,0,1\n",
       "Path: .;\n",
       "\n",
       "Test: None, model=SequentialRNN(\n",
       "  (0): MultiBatchEncoder(\n",
       "    (module): AWD_LSTM(\n",
       "      (encoder): Embedding(7520, 400, padding_idx=1)\n",
       "      (encoder_dp): EmbeddingDropout(\n",
       "        (emb): Embedding(7520, 400, padding_idx=1)\n",
       "      )\n",
       "      (rnns): ModuleList(\n",
       "        (0): WeightDropout(\n",
       "          (module): LSTM(400, 1152, batch_first=True)\n",
       "        )\n",
       "        (1): WeightDropout(\n",
       "          (module): LSTM(1152, 1152, batch_first=True)\n",
       "        )\n",
       "        (2): WeightDropout(\n",
       "          (module): LSTM(1152, 400, batch_first=True)\n",
       "        )\n",
       "      )\n",
       "      (input_dp): RNNDropout()\n",
       "      (hidden_dps): ModuleList(\n",
       "        (0): RNNDropout()\n",
       "        (1): RNNDropout()\n",
       "        (2): RNNDropout()\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (1): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.4, inplace=False)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace=True)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1, inplace=False)\n",
       "      (6): Linear(in_features=50, out_features=2, bias=True)\n",
       "    )\n",
       "  )\n",
       "), opt_func=functools.partial(<class 'torch.optim.adam.Adam'>, betas=(0.9, 0.99)), loss_func=FlattenedLoss of CrossEntropyLoss(), metrics=[<function accuracy at 0x7f58083928c8>], true_wd=True, bn_wd=True, wd=0.01, train_bn=True, path=PosixPath('.'), model_dir='models', callback_fns=[functools.partial(<class 'fastai.basic_train.Recorder'>, add_time=True, silent=False)], callbacks=[RNNTrainer\n",
       "learn: RNNLearner(data=TextClasDataBunch;\n",
       "\n",
       "Train: LabelList (6090 items)\n",
       "x: TextList\n",
       "xxbos in xxup both ' xxunk and times of national emergency . ',xxbos xxunk ok i was n't completely xxunk i may have also been in a food xxunk bc of the xxunk / xxunk / xxunk i also annihilated w / xxunk,xxbos xxmaj ladies here 's how to recover from a # date you totally xxup bombed ... according to men http : / / t.co / xxunk http : / / t.co / xxunk,xxbos xxmaj the xxmaj latest : xxmaj more homes razed by xxmaj northern xxmaj california wildfire : xxmaj the latest on wildfires burning in xxmaj california andû _ http : / / t.co / xxunk,xxbos xxmaj it hurts for me to eat cause i burned my xxunk with a xxunk yesterday !\n",
       "y: CategoryList\n",
       "1,0,0,1,0\n",
       "Path: .;\n",
       "\n",
       "Valid: LabelList (1523 items)\n",
       "x: TextList\n",
       "xxbos xxmaj students at xxmaj xxunk remember xxmaj australian casualties at xxmaj xxunk xxmaj xxunk xxmaj gallipoli \n",
       "  http : / / t.co / xxunk via xxunk,xxbos xxmaj israel wrecked my home . xxmaj now it wants my land . \n",
       "  https : / / t.co / xxunk,xxbos ' xxunk : xxmaj haha jam xxunk xxunk garden city xxunk bumper to bumper with xxup xxunk xxunk decide to chill via xxunk ' xxunk siren xxunk,xxbos i feel like i 'm drowning inside my own body ! !,xxbos xxmaj police xxmaj officer xxmaj wounded xxmaj suspect xxmaj dead xxmaj after xxmaj exchanging xxmaj shots : xxmaj richmond police officer wounded suspect killed a ... http : / / t.co / xxunk\n",
       "y: CategoryList\n",
       "1,1,0,0,1\n",
       "Path: .;\n",
       "\n",
       "Test: None, model=SequentialRNN(\n",
       "  (0): MultiBatchEncoder(\n",
       "    (module): AWD_LSTM(\n",
       "      (encoder): Embedding(7520, 400, padding_idx=1)\n",
       "      (encoder_dp): EmbeddingDropout(\n",
       "        (emb): Embedding(7520, 400, padding_idx=1)\n",
       "      )\n",
       "      (rnns): ModuleList(\n",
       "        (0): WeightDropout(\n",
       "          (module): LSTM(400, 1152, batch_first=True)\n",
       "        )\n",
       "        (1): WeightDropout(\n",
       "          (module): LSTM(1152, 1152, batch_first=True)\n",
       "        )\n",
       "        (2): WeightDropout(\n",
       "          (module): LSTM(1152, 400, batch_first=True)\n",
       "        )\n",
       "      )\n",
       "      (input_dp): RNNDropout()\n",
       "      (hidden_dps): ModuleList(\n",
       "        (0): RNNDropout()\n",
       "        (1): RNNDropout()\n",
       "        (2): RNNDropout()\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (1): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.4, inplace=False)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace=True)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1, inplace=False)\n",
       "      (6): Linear(in_features=50, out_features=2, bias=True)\n",
       "    )\n",
       "  )\n",
       "), opt_func=functools.partial(<class 'torch.optim.adam.Adam'>, betas=(0.9, 0.99)), loss_func=FlattenedLoss of CrossEntropyLoss(), metrics=[<function accuracy at 0x7f58083928c8>], true_wd=True, bn_wd=True, wd=0.01, train_bn=True, path=PosixPath('.'), model_dir='models', callback_fns=[functools.partial(<class 'fastai.basic_train.Recorder'>, add_time=True, silent=False)], callbacks=[...], layer_groups=[Sequential(\n",
       "  (0): Embedding(7520, 400, padding_idx=1)\n",
       "  (1): EmbeddingDropout(\n",
       "    (emb): Embedding(7520, 400, padding_idx=1)\n",
       "  )\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(400, 1152, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1152, 1152, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1152, 400, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.4, inplace=False)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace=True)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1, inplace=False)\n",
       "      (6): Linear(in_features=50, out_features=2, bias=True)\n",
       "    )\n",
       "  )\n",
       ")], add_time=True, silent=False)\n",
       "alpha: 2.0\n",
       "beta: 1.0, SaveModelCallback\n",
       "learn: RNNLearner(data=TextClasDataBunch;\n",
       "\n",
       "Train: LabelList (6090 items)\n",
       "x: TextList\n",
       "xxbos in xxup both ' xxunk and times of national emergency . ',xxbos xxunk ok i was n't completely xxunk i may have also been in a food xxunk bc of the xxunk / xxunk / xxunk i also annihilated w / xxunk,xxbos xxmaj ladies here 's how to recover from a # date you totally xxup bombed ... according to men http : / / t.co / xxunk http : / / t.co / xxunk,xxbos xxmaj the xxmaj latest : xxmaj more homes razed by xxmaj northern xxmaj california wildfire : xxmaj the latest on wildfires burning in xxmaj california andû _ http : / / t.co / xxunk,xxbos xxmaj it hurts for me to eat cause i burned my xxunk with a xxunk yesterday !\n",
       "y: CategoryList\n",
       "1,0,0,1,0\n",
       "Path: .;\n",
       "\n",
       "Valid: LabelList (1523 items)\n",
       "x: TextList\n",
       "xxbos xxmaj students at xxmaj xxunk remember xxmaj australian casualties at xxmaj xxunk xxmaj xxunk xxmaj gallipoli \n",
       "  http : / / t.co / xxunk via xxunk,xxbos xxmaj israel wrecked my home . xxmaj now it wants my land . \n",
       "  https : / / t.co / xxunk,xxbos ' xxunk : xxmaj haha jam xxunk xxunk garden city xxunk bumper to bumper with xxup xxunk xxunk decide to chill via xxunk ' xxunk siren xxunk,xxbos i feel like i 'm drowning inside my own body ! !,xxbos xxmaj police xxmaj officer xxmaj wounded xxmaj suspect xxmaj dead xxmaj after xxmaj exchanging xxmaj shots : xxmaj richmond police officer wounded suspect killed a ... http : / / t.co / xxunk\n",
       "y: CategoryList\n",
       "1,1,0,0,1\n",
       "Path: .;\n",
       "\n",
       "Test: None, model=SequentialRNN(\n",
       "  (0): MultiBatchEncoder(\n",
       "    (module): AWD_LSTM(\n",
       "      (encoder): Embedding(7520, 400, padding_idx=1)\n",
       "      (encoder_dp): EmbeddingDropout(\n",
       "        (emb): Embedding(7520, 400, padding_idx=1)\n",
       "      )\n",
       "      (rnns): ModuleList(\n",
       "        (0): WeightDropout(\n",
       "          (module): LSTM(400, 1152, batch_first=True)\n",
       "        )\n",
       "        (1): WeightDropout(\n",
       "          (module): LSTM(1152, 1152, batch_first=True)\n",
       "        )\n",
       "        (2): WeightDropout(\n",
       "          (module): LSTM(1152, 400, batch_first=True)\n",
       "        )\n",
       "      )\n",
       "      (input_dp): RNNDropout()\n",
       "      (hidden_dps): ModuleList(\n",
       "        (0): RNNDropout()\n",
       "        (1): RNNDropout()\n",
       "        (2): RNNDropout()\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (1): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.4, inplace=False)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace=True)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1, inplace=False)\n",
       "      (6): Linear(in_features=50, out_features=2, bias=True)\n",
       "    )\n",
       "  )\n",
       "), opt_func=functools.partial(<class 'torch.optim.adam.Adam'>, betas=(0.9, 0.99)), loss_func=FlattenedLoss of CrossEntropyLoss(), metrics=[<function accuracy at 0x7f58083928c8>], true_wd=True, bn_wd=True, wd=0.01, train_bn=True, path=PosixPath('.'), model_dir='models', callback_fns=[functools.partial(<class 'fastai.basic_train.Recorder'>, add_time=True, silent=False)], callbacks=[...], layer_groups=[Sequential(\n",
       "  (0): Embedding(7520, 400, padding_idx=1)\n",
       "  (1): EmbeddingDropout(\n",
       "    (emb): Embedding(7520, 400, padding_idx=1)\n",
       "  )\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(400, 1152, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1152, 1152, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1152, 400, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.4, inplace=False)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace=True)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1, inplace=False)\n",
       "      (6): Linear(in_features=50, out_features=2, bias=True)\n",
       "    )\n",
       "  )\n",
       ")], add_time=True, silent=False)\n",
       "monitor: accuracy\n",
       "mode: auto\n",
       "every: improvement\n",
       "name: best_clas], layer_groups=[Sequential(\n",
       "  (0): Embedding(7520, 400, padding_idx=1)\n",
       "  (1): EmbeddingDropout(\n",
       "    (emb): Embedding(7520, 400, padding_idx=1)\n",
       "  )\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(400, 1152, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1152, 1152, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1152, 400, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.4, inplace=False)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace=True)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1, inplace=False)\n",
       "      (6): Linear(in_features=50, out_features=2, bias=True)\n",
       "    )\n",
       "  )\n",
       ")], add_time=True, silent=False)"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.load('best_clas')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEGCAYAAAB/+QKOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOydeXhU1d34P2e27CEhCbIECPu+R8AFBBXFDdeqWNuqrfa1tS61C3ZR625tLW3117fqq3ZTtFpbLCBuiIoioAKyE/Y9IQGyZ7bz++POvXNn5s6SZIZs5/M898nce8+9c2aSnO/97kJKiUKhUCgU4djaegIKhUKhaJ8oAaFQKBQKS5SAUCgUCoUlSkAoFAqFwhIlIBQKhUJhiaOtJ5AsCgsLZUlJSVtPQ6FQKDoUn3/++VEpZZHVuU4jIEpKSlizZk1bT0OhUCg6FEKIPdHOKROTQqFQKCxRAkKhUCgUligBoVAoFApLOo0PQqFQdB48Hg/79++nsbGxrafSaUhPT6e4uBin05nwNUpAKBSKdsf+/fvJycmhpKQEIURbT6fDI6WksrKS/fv3M2DAgISvUyYmhULR7mhsbKSgoEAJhyQhhKCgoKDZGpkSEAqFol2ihENyacn3qQSEiS2Hq1m1q6qtp6FQKBTtAiUgTMye/xFX//nTtp6GQqFoYyorKxk/fjzjx4+nZ8+e9OnTx9h3u90J3ePGG29k69atKZ5palFOaoVCoQijoKCAtWvXAnD//feTnZ3Nj370o5AxUkqklNhs1s/ZL7zwQsrnmWqUBqFQKBQJUlZWxsiRI/n617/OqFGjOHToELfccgulpaWMGjWKBx54wBh75plnsnbtWrxeL3l5ecybN49x48Zx2mmnUV5e3oafInGUBmGB3y+x2ZSDTKFoD/zqzY1sOlid1HuO7J3LfZeMatG1W7Zs4a9//SulpaUAPPbYY3Tv3h2v18vMmTO56qqrGDlyZMg1J06c4KyzzuKxxx7jhz/8Ic8//zzz5s1r9edINUqDsMCn+nQrFIooDBo0yBAOAC+//DITJ05k4sSJbN68mU2bNkVck5GRwQUXXADApEmT2L1798mabqtQGoQFPr/EaW/rWSgUCqDFT/qpIisry3i9fft2fv/737Nq1Sry8vK4/vrrLXMNXC6X8dput+P1ek/KXFuL0iAs8PmVBqFQKOJTXV1NTk4Oubm5HDp0iKVLl7b1lJKK0iAsUCYmhUKRCBMnTmTkyJEMHz6c/v37c8YZZ7T1lJKKkJ1kMSwtLZWtbRhUMm8RAF/+chb5Wa44oxUKRarYvHkzI0aMaOtpdDqsvlchxOdSylKr8Sk1MQkhZgshtgohyoQQES57IcQNQogKIcTawPadwPHxQohPhRAbhRDrhRDXpHKe4SgNQqFQKFJoYhJC2IGngVnAfmC1EGKhlDLcxf+KlPK2sGP1wDellNuFEL2Bz4UQS6WUx1M1XzN+5YNQKBSKlGoQk4EyKeVOKaUbWABcmsiFUsptUsrtgdcHgXLAsql2KvAqAaFQKBQpFRB9gH2m/f2BY+FcGTAjvSaE6Bt+UggxGXABOyzO3SKEWCOEWFNRUZGseasoJoVCoaDtw1zfBEqklGOBd4C/mE8KIXoBfwNulFL6wy+WUj4jpSyVUpYWFSVPwfArH4RCoVCkVEAcAMwaQXHgmIGUslJK2RTYfQ6YpJ8TQuQCi4CfSylXpnCeAGw/UmO8ViYmhUKhSK2AWA0MEUIMEEK4gGuBheYBAQ1BZw6wOXDcBbwB/FVK+VoK52gw63cfGq+Vk1qh6NrMnDkzIult/vz53HrrrVGvyc7OBuDgwYNcddVVlmNmzJhBvHD8+fPnU19fb+xfeOGFHD9+UuJzIkiZgJBSeoHbgKVoC/+rUsqNQogHhBBzAsNuD4SyrgNuB24IHL8amA7cYAqBHZ+quYajNAiFomszd+5cFixYEHJswYIFzJ07N+61vXv35rXXWv5cGy4gFi9eTF5eXovv1xpS6oOQUi6WUg6VUg6SUj4cOHavlHJh4PU9UspRUspxUsqZUsotgeN/l1I6pZTjTdvaVM7VjHJSKxRdm6uuuopFixYZzYF2797NwYMHmTBhAueccw4TJ05kzJgx/Oc//4m4dvfu3YwePRqAhoYGrr32WkaMGMHll19OQ0ODMe7WW281yoTfd999APzhD3/g4MGDzJw5k5kzZwJQUlLC0aNHAXjyyScZPXo0o0ePZv78+cb7jRgxgptvvplRo0Zx3nnnhbxPa1ClNizYdLCa0X26tfU0FAoFwJJ5cPir5N6z5xi44LGop7t3787kyZNZsmQJl156KQsWLODqq68mIyODN954g9zcXI4ePcrUqVOZM2dO1H7Pf/rTn8jMzGTz5s2sX7+eiRMnGucefvhhunfvjs/n45xzzmH9+vXcfvvtPPnkkyxbtozCwsKQe33++ee88MILfPbZZ0gpmTJlCmeddRb5+fls376dl19+mWeffZarr76a119/neuvv77VX1NbRzG1S37y+nqqGz1tPQ2FQtGGmM1MunlJSsnPfvYzxo4dy7nnnsuBAwc4cuRI1Ht8+OGHxkI9duxYxo4da5x79dVXmThxIhMmTGDjxo2WZcLNfPzxx1x++eVkZWWRnZ3NFVdcwUcffQTAgAEDGD9es8Ins5y40iCisOdoPWOKlRahULQ5MZ70U8mll17KXXfdxRdffEF9fT2TJk3ixRdfpKKigs8//xyn00lJSYllee947Nq1i9/85jesXr2a/Px8brjhhhbdRyctLc14bbfbk2Zi6vIaRIPbx3Mf7Yw4HkVjVCgUXYTs7GxmzpzJTTfdZDinT5w4QY8ePXA6nSxbtow9e/bEvMf06dN56aWXANiwYQPr168HtDLhWVlZdOvWjSNHjrBkyRLjmpycHGpqaiLuNW3aNP79739TX19PXV0db7zxBtOmTUvWx7Wky2sQ9W4vDy3aHHHcpiSEQtHlmTt3Lpdffrlhavr617/OJZdcwpgxYygtLWX48OExr7/11lu58cYbGTFiBCNGjGDSJC3Va9y4cUyYMIHhw4fTt2/fkDLht9xyC7Nnz6Z3794sW7bMOD5x4kRuuOEGJk+eDMB3vvMdJkyYkNLudF2+3HdVnZuJD74TcXzJHdMY0Ss3GVNTKBTNRJX7Tg3tqtx3RyCanqAUCIVC0dXp8gIimilJRBUdCoVC0TXo8gIimhxQGoRC0bZ0FvN3e6El32eXFxC2KIJAVXRVKNqO9PR0KisrlZBIElJKKisrSU9Pb9Z1XT6KKVoGpD+iuLhCoThZFBcXs3//fpLZ56Wrk56eTnFxcbOu6fICQmkQCkX7w+l0MmDAgLaeRpeny5uYojmjlYBQKBRdHSUgomoQJ3ceCoVC0d5QAiKKgFAlvxUKRVdHCYgoJiYVPaFQKLo6XV5ARHNSKw1CoVB0dbq8gIga5qrkg0Kh6OJ0eQGhwlwVCoXCmi4vIKJrEEpAKBSKrk2XFxDRUCYmhULR1VECIgp+JSEUCkUXRwmIKHQVE1NNo4cGt6+tp6FQKNohSkBEoauEuY65/23OePz9tp6GQqFoh6RUQAghZgshtgohyoQQ8yzO3yCEqBBCrA1s3zGd+5YQYntg+1Yq52nmG1P7A13LB1FV527rKSgUinZIyqq5CiHswNPALGA/sFoIsVBKuSls6CtSytvCru0O3AeUAhL4PHDtsVTNVyc7XftKuoKJyZwtXu/2kunq8sV9FQqFiVRqEJOBMinlTimlG1gAXJrgtecD70gpqwJC4R1gdormGYKeF9HZBYTb62fAPYuN/Q+3qbr7CoUilFQKiD7APtP+/sCxcK4UQqwXQrwmhOjbnGuFELcIIdYIIdYkq7GI3qO6s/sgfv/etpD9nHRnUu/v90s2HjzB/mP1Sb2vQqE4ebS1k/pNoERKORZNS/hLcy6WUj4jpSyVUpYWFRUlZULnj+oZuHdSbtdu2V0ZunCXldcm9f7//HwfF/3hY858fFlS76tQKE4eqRQQB4C+pv3iwDEDKWWllLIpsPscMCnRa5OJnkz9swuHk3OSfBDVjR4e+u8mGj1tE2Ianudx38KNSb3/9iPJFTgKheLkk0oBsRoYIoQYIIRwAdcCC80DhBC9TLtzgM2B10uB84QQ+UKIfOC8wLGUIIyf4qSZmJ58exvPfbyLhWsPpvR9ouHxpbbpdq+8jJTeX6FQpJ6Uha1IKb1CiNvQFnY78LyUcqMQ4gFgjZRyIXC7EGIO4AWqgBsC11YJIR5EEzIAD0gpq1I1Vx0hwBbwUqfaxHSsXgstddijVAtMIlJKapq85Jr8DE3e2ALib5/uZlBRNqcPLmzRe67ddzzk/aPVvFIoFO2XlMY1SikXA4vDjt1ren0PcE+Ua58Hnk/l/HSEEIZE0KOYfCmWEHVNXgCc9tS7gV78ZDe/enMTH/1kJn27ZwLQ5IkUEPuP1VOcr53/5X80k9Puxy5q0Xu+uS6oGe2tqqd/QVaL7qNQKNqOtnZStwvMz7b2wJPu4RONKX3PhoDvoeEk+CDe3XwECDqiy8prWbU7UiE7Xu+JOHYsCUl0L6zY3ep7KBSKk48SECaEEIa0+P1721P6XrqP4yevreetDYdS+l66lnLji6t5YumWqDkPNY3eiGN/Wr6j1e9fkOVq9T0UCsXJRwkIglFMAk5aNrHZgvXqmv0peY9Gj48T9R4ctuCv+ellOyivaQoZ99LNUwCtcF847ji+imiU9s/n9EEFpDttnGiIvK9CoWj/qNoKaNFLWkUPyE5zUJjtYuawHil9T7OAqHdHPrm3hk93VPLpzkpW76ri052VXDimZ8j5/zVpBct+NMPwu8x/dzuzRp4S4lBubqXXvZX13PqPz9l1tI7Sku50y3Dy3Me7qKpz8+Q141v+oRQKxUlHaRBgmJX0dTHdaU95mKskeP9kl9ue++xK/vDedj7dWQnA4q8ORx07oDCL7DTtOWHToWqG/fIt/rZyj3G+2kKriMWH2yvYeLCaercPp03QLUOLnPrXlylLY1EoFClCCQgT+nOz027Dm0IBIaVk9e5g3cH6NurH8P2Zg4BggULQTEq//PcGY7+5AsLlCP5J2U0CQqFQdDyUgMCUKBdQIew2kVIN4q0NoU/0bSUg7p41DIA0hz3qmBVllQnf75MdR/nJa+uNfafdFiIg2iprXKFQtAwlIAialnQcNoHXn7pM4x0VoWUostKiL9DhfLC1nPX7j8cfmAB6UmA8EnFU1zR6uO7Zz0KOOeyCNGfws13/3GfhlykUinaMEhAmdEGRag1iQGE2gOE8bk5O3g0vrGbOUytCjkkpWbWrKm7uxj0XDOemMwZYnvv1lWOjXhdrYf/P2gMsWn+IMfe/HXEuO81hlC4BWLMn5e08FApFElECAj2KKWhq0jSI1AkIPUv7h7OGcu2pfdleXsvOipYXt3vxk91c/edPmf7EsqiC7ddXjuW7Zw1i6sDuluevPrWv5XHAMqlO544Fa/n+S19YnuuW4YwoCqhQKDoOSkAQaWKy2wReX+oWNt1k47LbDTPP2b9d3qzWn79ZutV4rUcdub3+CP+GztdKiwHwNONzfftMa20jUaobPYwp7hZyrLP32VAoOhMqDwJTqY2ApHDYbSn1QejlNdKcNrJcQRv9irKjLN14mMevHEtWWuSvpskbdPI+tayMH84ais0mQuoq/d0UoqozoDDLcMCfNayIMwYX8NBlYyLGvXDDqWw4cAK/hFkjT4nwlTSX704fRJ+8DCb0zeOaZ1YC2mfPtvhsCoWi/aH+U03ogqKsvJaqOjd+v0zYkdsc9hytI81hozA7jXSTE/dXb27iaG0Ts0aewqXjI5vvhZfCONHgIT/LhdtUujtcsP3g7MH84Owhxn52moN/fGeq5bxmDu/BzOHBBMG9VbG7wckYzpP7LxlpFAacMrCAhy4bzS/+vYH6Jq8SEApFB0GZmAiGt+qmJt3Uc/BEQ0rer7rRQ36mC7tNhAgIPQx025EannxnW4g5Zl9VPY8s2hxyH11gNJnCR835FaCZy8y5Cc3B5YgtHKNZi3rkpHFDmDNcj9Sqa6OQXoVC0XzUoxyh1VxDjqeoh4G59lJeZjBPoDZQAvzpZVopDL9f8qPztVyFab+ObN2pm6pG9+nGJzus8xVaY/KPV4o8mhnu+RtOjTim17jSy5wrFIr2j9IgTIgwUZEK8fDnsOqoX5sUPXroqWVlMe+l13AKdwQD9AuYd1rzGWIl0IG1w/my8b0Z3SdyPlkBAdFWSYEKhaL5KAEBEbWYdFIRb/Poki0A5AbKW7gcNrY9dEHMa6JF/ug1nKwS2WaP1nIsbK3Qgkr758ecg1UocE66dWmNTMPEpDQIhaKjoAQE5p7UofhSGOp6jSnvIJ6PoDpKuex9xzQnslV/6UFFWge3fgUt7w1tswl+efFIILQUeF2Tl+P1bsvvZ+gp2Zb3MjSIJqVBKBQdBeWDILqvIZWhroea0bEuWj+Fo7WaM91Kg7i6tC/F+ZmcPqigZRMMkBeopXS83kNeptb459wnl3PoRCOrf36uMW5Mn25cOr4310/tb3mfTJfSIBSKjoYSECbC5URrk7rcXj/VjR4Ks9MAQrKK9WPhXF1aHNFAKJqA0J3aHp+kX/dM/jh3AsX5GUg0oXfG4MJWzR+CTnTzHHThtsaUYZ2X6eQ70wZGvY8uIJJd2lyhUKQOZWLC3FEuVEK0ttzGj19bR+lD7xqC4eOyo8a5eRcMt7zmvJHB5j666SmaY/dPH+zgzXUHcXv9OO2CcX3zKMhOiyp8WoJejfXg8ciQ31v/ESyxES/iS0/8UxqEQtFxUAKCSN/D+aNOAVqvQfxn7UEAmgImoCPV2pP3W3dOC8l/AHjztjOZf8140pzBX4nb68fnl5Y+Bp0fvPwlH5cdxRUn4qilnJKbDoQKAyviucLTHDZsQvkgFIqOhBIQZgKr3LWn9gNar0Ho6PkKPw70SugZWHTNjCnuxmUT+kTUY1q47oDhY/jzNybx7g/P4rSBoX6FEw0eysprkjLXcPRsaICSeYsAa6d6vIRzIQRZLodl32uFQtE+UQKCoKbgsAUbBmnHW+6kNhfTa/T4Qmz4sUpNmE1MAHe9ss7QIPoXZDK4RzYv3zI1YkFuThG+5tI9yxWyb+UUnzO+d9z79MhN48t9yellkQoeXbKZl1ftbetpKBTthpQKCCHEbCHEViFEmRBiXoxxVwohpBCiNLDvFEL8RQjxlRBisxDinlTOUy8p5AhkDuuCojUVXc1Jbmv2HAspfOeIkaGc4Yo0Fem1llym605mUdTLJwTrQs38zQch53LSHJQ9fAGXTyiOe58zBxeyq6Iu7jgpZcw6T6niz8t3cs+/vjrp76tQtFdSJiCEEHbgaeACYCQwVwgx0mJcDnAHYO5K8zUgTUo5BpgEfFcIUZKquer9GSI1iOQsUrWN3lYlrOlP7PFKX6QKc8XZXUdDF/j8LFdMgRdynzQHjd7YPoiKmiYG3LOYAfcsVqXBFYo2JpUrzmSgTEq5U0rpBhYAl1qMexB4HDAnBkggSwjhADIAN1Cdqon6AwJCFwwOe0CDaOEC5Q1zKh9vcBtlMTItNIRwlt45naeum2DsGxqEyfY/d3K/Fs2tJdht0f9M4lV8NZPutOPxxXa66458CIbxKhSKtiGVAqIPsM+0vz9wzEAIMRHoK6VcFHbta0AdcAjYC/xGShnR1kwIcYsQYo0QYk1FRUWLJ6q7GoIahPa1tPQJ9t6FG0P2G90+I/7/5ZutS22bGdYzh4vH9mZc3zz65GXg8UaamIb3zGnR3FrCqt3WhQCbS0YgcqvRE12L0IUzhOZZpJpq5TxXKCJoMye1EMIGPAncbXF6MuADegMDgLuFEBFZWFLKZ6SUpVLK0qKiohbPRc+YjvBBtFBAvP55aKJbk89v5DLoZa8TYVTvXJq8PkODcJo0iG+EZSynUmCcM/yUpNwn3aULiOgahNkBvnDdwaS8byLsKA/6iNbtO07JvEWUlbeuYZJC0dFJpYA4AJhLlRYHjunkAKOBD4QQu4GpwMKAo/o64C0ppUdKWQ6sAEpTNVFdDiQriiktLAy0yeM3TEwZrsST1zOcdho9fiNCyaxBmBsZfXX/efz7+2e0aK6JcMPpJXz805mW5+44Z4jlcSsS0SDMAqIuiTkTVXVuy+grHbN/59KnVwDw/pYjSXt/haIjkkoBsRoYIoQYIIRwAdcCC/WTUsoTUspCKWWJlLIEWAnMkVKuQTMrnQ0ghMhCEx5bUjhXwOSDaKUGYe7KBpoPQdcgMp2JaxDpThsNHh8vrNgNgNMe6uh+8cZT+eXFI8lJd0Yk3iUTm01QnJ/J/GvGG8fuOncoZw0t4nszByV8n/RAEmBDDAHRZFrE3918hAMWGdwtYeKD7/D9l6In+1kFTf15+c6kvLdC0VFJmYCQUnqB24ClwGbgVSnlRiHEA0KIOXEufxrIFkJsRBM0L0gp16dqrjrJimKyh0UsNbh9hoCwCmONRobTjs8vOVrbBESWs5gxrAffPnOA1aUp4TJTuOv4fnn85abJcXtGmGmuBgFQXp14UcNo6EED72wK1QgaPT6WbS0HgpFsZirr3CqxT9GlSakPQkq5WEo5VEo5SEr5cODYvVLKhRZjZwS0B6SUtVLKr0kpR0kpR0opn0jlPHVshgahfS0tzYPw+rXied+fOYj+BZlsO1JDg9uHTUSan2KRSq2gtbQkaFcXjrGik3QN4sHLRgNwPEqhwkRZ/NUhBv98ieW5RxZv5sYXVrN+//GoDwPn/HZ5q95foejIqExqE/qiZzfCXBP3QSxcd5AZTyzD55f4/BKXw8aPzx9Oj5w0Nh6s5t3NR8hyOZrVxrQ52sbJYlyge11LQlB75GhFBMurm6KO0R3yBYHs7dbUbmry+vhejBpS+wIhuuXVTUaoczjlNdHnqlB0dpSAMKEv3s6AJtGc8hW3v/wluyvrqXN78fr9hrlKzxPYcrim2Qt+eooK8LWGm6drwWSDe1g3BopFQZYmIMLrTZnRTUx6mfHWVH+tsFjczeXGdfNYo9cXUopdoVBoKAFhQn+21yNawhPeEsHrk3h90vBjnG0KEe3ZLbJIXyzaowZx8djebH5gNkNPaX5YrVHyO4b2YQiIDE2DMC/oy7dVUDJvES+u2MUf3tsetxxHk0XU0oh73zJe6zkXPr/kcBJ8HQpFZ0MJCBO69UdfOFpSAK+uyct7W8rZeFBL/L5/TrC6yKjeuc26V4bJBzFrZHJyEZJBSwWXy2HD5bBRG0MrcAdKcegahLkXxkP/3QTA/W9u4sl3tlEXp/lQTWNs7UMvf+LzS+5YsBaAx64YE+dTKBRdByUgTOgNg3QNwtOCPIjKMPNJmsNuhKc2t5GPuTfEs99MWRrISSU7zRFTg2gMPPXnZjgRAhpMwmRCv7zQsTGioQC2H4ldAl1PJTHPp1/3TPqZSpwrFF0ZJSAsCJqYWqZBABTnZxjHdMHT3CfvjHYcxdRSstLsMRPgzPkiGU67se/zy4hWrPHal5ZVxM6E1qPWfvkfU2kUAR/+JJgU2Bl/BwpFoigBYUI3MdltAiGIWVRO551NR4w8BQhG99x57lDjmB6Zk9WMLGponz6I1pLhtLMzxsLd4PaS4bRjswkaPT6e+3gXlbVNVNZGOpzjaRBNMUp6AJYVdnWz4l9vmsxFY3qF1IZSKLoaCQkIIcQgIURa4PUMIcTtQoi8eNd1ZJx2W1wfRJPXx81/XcNlgdIMgNEYyKrrWnMX/D55mhZyTlhmdkdm25Fa1u0/EfXpv87tMyre6oFFkx56l/e2lEeM1bWLT3dUssCi0U+T109RTqRZL1bwgX5u+tAiirtnUNPoVclyii5LohrE64BPCDEYeAatxtJLKZtVG2F+oHTaRNwoJj1KZv+xYDmIhxdtBkLrJp05uBBIrNS3mZx0J+/+cDp/NJX+7ujo3/HxButQ17Ly2pA2pzp6I58LRgc77uklO+Y+u5J5Fo1+3F5/yO9BZ3dlnXE+HPNDQaZT0/hmz//Icq4KRWcnUQHhD5TOuBz4o5Tyx0Cv1E2rbRCm/GCH3RbXxOSxWGCCGkTwXvqi2FwBATC4Rw6ZzTRNtWeemjsRiB5hVFZey4he0aO99lQG+0/EqukEmmnP5bCx8LYz+OPcCcZ9L3v6E6SUNFk0LyrMDrZXLQi8TlY9KIWio5HoyuMRQswFvgVcEjjmTM2U2o4QDcJuwxMnecodQ4C47EFh4Dc61imXT26G9idXHaWEhsfrj+kYNmc8v7PpCDOHRTe/ub0+XHYbY4vzGFucx96qejYfqqa2ycuAexaH9PUe1TuXX80ZRWlJd+NYTnrnEcwKRUtIdMW6ETgNeFhKuUsIMQD4W+qm1fY47fFNTB5vdAFi5YPQY/u7Mjnp2ncQ3qBnR0UtJfMWUdPkNRzDT1mY1l6/9XSmDdFMdg5bqAM53GTk9vpDfg/h483yv8HtCxEOYF3hVaHoSiT0iCSl3ATcDiCEyAdypJSPp3JibUG4BnHoRCNSyqj1kzYdit4F1bwwPXbFWF5ds4/Rvbslba4dldzAU/mxulABsXj9IeO1noV+8djejO2TR7rTxuRH3gO0bOy/fXsKEx98J6J+UoPbF/K96yYmnVh9wXeG9dqGlpd7Vyg6CwkJCCHEB8CcwPjPgXIhxAop5Q9TOLeTjtkHsbeqnr1V9fzriwNcOanYcvzrX+y3PA6hvRv6ds/k7vOGJW+iHZji/EzsNmE4inXys4K2f/OTfr8CzWH99HUTKa8JlsOwyqeo93jpZrJ8erwy5PfQLaN5GlxLSq0oFJ2JRE1M3aSU1cAVwF+llFOAc1M3rbbB6gHz873Hoo4/a2iwzelvvjYu5Fxzynp3JVwOGz6/5I/vl4XUUrKbhILdFvmLuGhsL248I9j7IssVmZFdHxY62+Tz4zIVPLQS9LF+T7pwUii6KomuYg4hRC/gauC/KZxPm2JlgIhVUdWcqFUfVl/I7KRWWGMupmfuxxDuK7Ai02Xn7U1HKJm3yDgWnlsRHuZqJXjMWdPhnD6okDMHF5LutMUtDKhQdEYSFRAPoHWG27gamooAACAASURBVCGlXC2EGAhsT9202gYrDSLdGf0rMi9ItU1ew3k6rm8evfKaV7m1K2J+4jf7ExKx7OiVYaPdD7QopniaXG7Aaa4nJYYzY1gRjR4/1Q0tLzuuUHRUEnVS/xP4p2l/J3BlqibVnoj13NhoiqOva/Lyt29PSf2EOhH7j9Vz4wur+MXFI0M0iOdX7OKOc4fEvNaqbEm4FhfupAZY9bNzuG/hRpZsOAxoDwBL7pjGKbnWAj0/U/ONHG9w001FoSm6GImW2igWQrwhhCgPbK8LIaw9tx2aoArxz/85DYjdl7rBrT3qju6Ty3VT+qd2ap2QD7dVsG7/Cf6xck/I93wigTajVhpEuIlpX1VDRNvWHrnp/On6Sca+EIIRvXLpbnKSm8kORF21pIOeQtHRSdTE9AKwEOgd2N4MHOtUmE1Mp5Z0JzvNEbOia6PXR2G2i//+YFpUE4UiOnpZi/Ka0Jaf/RNwDmelRfp4qurdlMxbxL++2M+yrVrtppctajQ1h5yAIKqN01tCoeiMJJoqWiSlNAuEF4UQd6ZiQm1JuAvCbhP4YvSEaHT7Ip5QFfGZOrA7K3dW8d6WI4DmOzD7Hf5wbfzaU1YahN7e9bdvb4tbHuP9u8/iWH18TUVpEIquTKIaRKUQ4nohhD2wXQ9UpnJibUF4QpzDJmImSx1v8JBtsVApYnPPBSMA2HBASzRscPsMDeKJq8Yyrm/8QsGZFoJZNzGZI46iVcIdWJTNpP75cd9H//0qAaHoiiQqIG5CC3E9DBwCrgJuSNGc2gwrDSI8W9fMvqp6SgqyUjupTki4eajO7TV8EJdP6JPQPaxKp6/cqT2zHDwRTKi7dnK/lk4TCAqIeO1LFYrOSEICQkq5R0o5R0pZJKXsIaW8jC4QxeSwiag+iK2Ha9heXktNk+oV0FwywiKQ/H5pCAirXAUrrEx7245ENiJqbf6CMjEpujKtSfeNW2ZDCDFbCLFVCFEmhJgXY9yVQggphCg1HRsrhPhUCLFRCPGVECLliQXheRB2u4gaxfT8x7sAWFHW6SxtKSfcPOT2+fFLiRCRZr6o90iwdHpr09synHZsAr46cKKVd1IoOh6tERAx/5OFEHbgaeACYCQwVwgx0mJcDnAH8JnpmAP4O/A/UspRwAwg5Y/qIuwj7atq4F9fHrAca0vwSVcRSWaYicll18pv2BMUDhDsFR3vktYmQAsh8EtYtP4Qn+w42rqbKRQdjNYIiHj/epOBMinlTimlG1gAXGox7kHgcaDRdOw8YL2Uch2AlLJSShm7O0wSaMb6ZPQSSKQshCKUNIedn184wtjPSnNQ7/Y1qyWrPnbKgO7seOTCkHOF2WlGc6AzA9ntyWCvqVmRon1Q7/byZYx6aYrWEVNACCFqhBDVFlsNWj5ELPoA+0z7+wPHzPefCPSVUi4ilKGAFEIsFUJ8IYT4SZT53SKEWCOEWFNRURFnOs3niol9ouY36LbyRG3milBunj6QX185lt7d0tleXsuuo3VG1nIi6D4In19itwmumxJ0Rjd6fPTrnsHwnjkpiTJ7/fP9vPFl9Eq+bcUrq/fy8zciW692Zn7+xgYu/3+fsOVw9NL7ipYTU0BIKXOklLkWW46UslX/eUIIG/AkcLfFaQdwJvD1wM/LhRDnWMzvGSllqZSytKioKPx0C+YUup/msEXtGqf7Jqx6HisS4+pT+zKsZw4Ay7dVkN+MUha6D6I4X0uqK8xOM87VNnlZUVaJM8m/G7fPz/YjNdz9z3Xc9cq6pN47Gfz09a/4x2etSwzsaCzfpj0Yzp7/UUjDqPe3HOGV1V3ru0gFqVzdDgB9TfvFgWM6OcBo4AMhxG5gKrAw4KjeD3wopTwqpawHFgMTUzhXINJB6rLbLBvbQ7A73M3TB6Z6Wp2aX1wcdEvlNUODGN27Gz8+fxi3n6PVbHLZQ393tU3ekF4QreG/PzgT0KrDzvrdh0m5pyI5DCgMhpmby7/f9OIafvp619KmUkEqBcRqYIgQYoAQwgVci1auAwAp5QkpZaGUskRKWQKsBOZIKdegVY4dI4TIDDiszwI2pXCulrgc0QVEQZb2xPrN01QNptYw0PQPvsuiq1s0bDbB92cONhaINIuy7F/sPd76CQKDe2QDoeXJ2zN+i8i75dsq+M3SrW0wm9QydWCwTaweiqxKsyePlAkIKaUXuA1tsd8MvCql3CiEeEAIMSfOtcfQzE+rgbXAFxZ+iqQT/rzpimFi8gZKcDiUialVmLW2ytqmFt8nWdqCFboZ8VidO+qYyQ+/S8m8Rdz4wqqUzQNg4D2LeO6jnTHH1Hsi4zm+9fwqnlpWZik8OiJNXh8l8xbx9LIdxjFdQHQUQd4RSOnqJqVcLKUcKqUcJKV8OHDsXinlQouxMwLag77/dynlKCnlaCmlpZM62UT6IOz4Aklcn++p4o4FXxr/YHrvAeWDSB5FOWnxB0XBmcIOfjabwGETMbOpy2s04bZsawVr9x3nSHUjJfMW8ea6g0mbh98v8Ut4aNHmmONe+mwPZzz2fkTHPdAKGnYGwiv3QtDEZBYQHtU2tlWo1c1EeB6E3kvA7fVzwwur+c/ag8Yiceh4I0U5aRH9BhQt56nrWu5mKrAo133JuHiBdonjtNuobgxNxWkK9AMJfyq/7OkV7KzQzGV/+3RP0uZgrgu29XAN+4/Vs8Eige+RxVs4cLzBKF64/1gwPLezlAzxWFQ4OB4ovmjOercSkorEUaubiXANQtcOmrw+bIGTvoB9s8HjI6sZcfuK+PTt3vIe0OeP6slDl43mje+dDsCQHtn8cW78qrCJ4nJECoh3N5VztLbJ0gz52udaGGwyS3SYs/r3VtVz5uPLuPiPH0cd7/b6eXTJZs58fJlxrKaxc5SGsfrOv/NXzQDxzPKg2amzCMS2QpUiNWHlgwDtH01Pd/AG/jC9fr/yPySZ1pjrhBBcP7U/Ww/XANAryf05XA4bx+pCF9fvv/QFw3vmsOCWqRHjX/9CExDeGOXim8O3nl/F8EBIMMDNf10TY7TGe1vK+fPyUH9FZ2mdGi14BEIfNFQNrdahBEQMdAHR5PUbCXG6fdPjkyqLOskkw1w39JRs7r9kJHPGJ1YVNlFcdhvbjtREHN96pIZ/fWFdjgWSsyBvOHCC5dsqjJh/K6wid5osnNWdRoMIExAT++VxpFrzA5l9WUpAtA4lIMxYJMoBLP7qkBFto6u2Xp8/6YlYXZXLxvfGYbclJStdCMENZwxIwqxCSXPYDB/AFRP7GELBYRMcj+H4PVLTGPVcovz27fjhqXUWTlsrO31nMbk0eUM/rx7S7Pb6Q0xxqhNg61ArnIlwJ7UuIB5dssUoJNfkCWoQqQyt7ErMv3YCv/nauLaeRkzM2s05w08xXnt8kj+8Xxb1umGn5EQ9lyhWC304Vgvh8yt2RRzTfSMdncqwkGO9j8j+Y/UhznylQbQOJSBMRBbrCx7Q1dZ9gYgQj0/5ILoSaaYS5d0yEi8J0mBh5mku8UI1pZQJm45W7a7qFLkQ4Vrb+aN6Alr4ubmHixIQrUOtcCbC5YNZjdW7oJ1o8ODzSzw+v9IguhB7KoNZ3rkZiVtmG5MgIPrkhzrcpwzoHuL/qm7wUhNYCB+4dBRPXh1bG9tT1fGr0nq8oUIu3aktZRf/8eOQPvKdxefSVigBYSK8FpO53ejKnVWA9g8/6GeL+WLvcRw29fV1Fcy2+9z0xDWIEw0ezv/dh7y/5UjEOY/PH2FLtyIjrMFSVpqDnt2C/bN2HK3lRIO2EI7qncsVE4st7zO5RCtLUV7der9IW+MJiw4zdxg0t5xt9KhEudagVrgYWEUo3vufjcZrpUF0He65YLjxujkmpkaPn61HarhzwdqIczOe+CAkRyEa4X3Ri7LT2H+swdj/1xf72XJIi7DqkRO98eK4vt0ADGHSkQlvBWwWon/6IJgHESscVhEfJSBMhC/3F43tFXO8sm92Hb5hKsrYnMZGOtWNXkrmLeI/a7Xop82HqjlwvIGKmvj1p8Lb3v7yktDGjH9fuZfH39oCxC5XcvmEYmMuHR2zX2bu5H6WPcohMtpJ0TyUgDAR7qSO9keno9TXroM5ic9hEzx13QTOGFwQMe6OQPlxgG9ZVPp9PRAeu8fUnc6qXIaZcB91dpqD3t2sNYVof7NZLju987RrqjuBBqFHdm15cDaPXjHG8EGYyUl3KA2ilSgBYSI8zNUKc80fVTWy62D2T9ltgovH9ubv357CqN5aa9OHLhsNwJUTi5k5TGtedeaQ6E2szKUiYpXLAOskuI9/eja/uGiExWiNJ64aG7JvswlyAr6TzmFiClRTDjjrLQWjhL98uidmnooiNkpAmLGQDz8+f1jIvjnG2ipTVdF50Utd6MJCCMFr/3M6n8w7m+un9mfzA7PpV5BpNGu3ioLWi8c158nWJ6WRk6NjswkumxA9W/yCMb3Iz3TynTODSYN2myAnzcHv39vO4q8OJfz+7RFP4P9QT64MT1q9aGwvI7Lrt29vO7mT60QoAWEiMg8CRvbKDdk3P30pDaJr8cp3TzO6y+lkuOz0DtR9CvdNWDUx2lFRCzSvDLXPL41Q1/NGBpP0YmXyZ6c5+PLe8/hBwOSlKyG5AQf79/7xRcLv3x6pbfSS6bIbwjq893g/Uz0mn2og1GJUqQ0TNgsJkWaybdoEmP2FygHWteiW4aRbn25xx827YDg9c9OZPKB7xDm/X+L2+jmagHPauEZK7EKw+7GLQo6HFzf83TWR+Q967+5bAq1xO0u3tb1VdSGhvi6Hjd2PXUTJPK2vWLpJOHeWz9wWKA3ChFXxvVG9ggtClitUnk4dGOmkVCiG98zlsSvH4rTbePjy0SHnHHYb1z/3Gb99J3Gzh99v/fBiLv+x8p5zjCglM067tnDqvbvNOQLh3PTiam5JoEpsW1FR08TX/vcTyqsb2VFRF1LdNhyz0zo8JFaROEpAmLASEN0ynYYzsNiktv7jO1N48urxJ21uio7J16f0Z2BRsO92VZ2bVburjP3e3dLpXxC7D4ZPSmwWf5vm4oY9o0Q1xSLcefv+lnLe3hSZ0Nde+Mdne1i9+xh/W7kHj88foiWEY3Zah4cJKxJHCQgT0aqJ3nB6Cb+/djzXTelnHDttYIHqJqdIiEn98qOemz60yLJ9phm/X1o6vEH723zk8jEJz+Xrpr/hjla4T7cUlZXX4vVJHBaJqj0s8kDq43y/iuioFc5EeKkNHYfdxqXj+5BpeiqxeqJTKKx48LLRPHHV2AiTyF3nDiXDZY8rIHwBH4QV988ZFfLgEg+zX6SjBVnoesCSDYcDtdAil6/Zo7WifT6/5PVbte6Cde6OnxjYVigB0QziJc4pFFakO+18rbRvyAPIwMIsbjyzhAynnZomL1sOV0e93uPzJ6VXBsCMoT0MQdXk8SGlZP6720Lev8nrY/3+40l5v2TSYFroowkI3Vfjl5JJ/fM5Y3CB6kvdCpSAaAYZLvV1KVqO2XH6u2vGk5vuNKKMZs//KOp1eyrrKc5veb9uM90ynbx153QynHYaPD6qG73Mf3c71z+3yhhz20tfMuepFZYd9NoSc2kbr9+6o6MuIHRzVJbLoUxMrUCteM0gllNMoYiHuaCcHj59tDZ2lm+T18f+Yw0hju5kkO600ejxcyzQeMccCvpOwFF9rK59ZSCv3x8sSVLv9lnmLQ0o1ASp7rTPSnOommmtQAmIZpDegiJtCoXOpP5BZ7WeRLfpUNC0Y1UCo75Je/ptTgXZROiW4aSq3m10Zgvv0AbRgzbainAT7z8tnOxfn9Kfv9w0mYsDhTaz0uwxBcTKnZXsrez4/TFSRUoFhBBithBiqxCiTAgxL8a4K4UQUghRGna8nxCiVgjxo1TO82/fnsxPZw+PO05pEIrWcOe5Q43XRjvbK4IRSH9fuSfiGt2RnGz/V3F+JovWH+LKP30SdUx7K0YZHq56X1hVW9CCR84aWmT4e7pnuowmX1Zc+8xKpj8Rv+R6VyVlAkIIYQeeBi4ARgJzhRARv1EhRA5wB/CZxW2eBJakao4604YUceuMQXHHqbBWRWswP5HrAmJgYdB05Pb6eX/LEV43PRnrHemsqpW2hkQ0kvoUR//c86+vuHPBlwmPD69fdfqgwrjX5GW6kNK6gu2i9R27HtXJIJUr3mSgTEq5U0rpBhYAl1qMexB4HAhJ8RRCXAbsAjZaXNMmhBdMUyhaiv6wYY5s6pOXwU0vruHuf66joqaJknmLeGXNPsC6rlNryEmPX2UnGf20Y/Hyqr38e+3BhMe7w+pXZaXF/wy65mUV0vuXT3Yn/N5dlVSueH2Afab9/YFjBkKIiUBfKeWisOPZwE+BX6Vwfs1GaRCKZGG14P/ry6DmoBf1e/7jXUDyNYhEqsm2t+gft9dPYXaw3H5WAj5B/aHOKhdCojKs49FmK54QwoZmQrrb4vT9wO+klLVx7nGLEGKNEGJNRUVFCmYZSnhxNIWipVi1q914MOiw1s1Rhg8iyRpEdWP8nhAnS0Dsq0rMSez2+o3opD55GVETW80cDvTffmTR5ohzHlWjKS6pXPEOAH1N+8WBYzo5wGjgAyHEbmAqsDDgqJ4C/Dpw/E7gZ0KI28LfQEr5jJSyVEpZWlQUvTlLsrBK7VcomsOFY7RMX/PidvcszXldY2oFGl6cLy3JGsTs0bHb6YJWUvtksCfBKCK3z09moGBmArIBgKO1WtXcT3dWhhz3+yVr97W/ZMD2RioFxGpgiBBigBDCBVwLLNRPSilPSCkLpZQlUsoSYCUwR0q5Rko5zXR8PvCIlPKpFM41IcKruSoUzeX3107gq/vPCzn2A1ObUp2XPtsbsp9sH8RVk4oto4DMnKzOczUJaDOgaRC6nz/R/0W9A2RemFO+VpXfSIiUCQgppRe4DVgKbAZelVJuFEI8IISYk6r3TSU2m2Bkr1wKs6M3hlcoYuG024zWn2bCG968/kVojH+yfRAQGcmUlxm2iDaFLtwrd1Zy4HhD0udRk2Aim9vrpyDwv3fuyB4JXXPLdC068YqJoaXQzeU3rAr8KTRS+kgspVwMLA47dm+UsTOiHL8/6RNrBYvvmNbWU1B0QtKdsRO6EonYaS77j4Uu9t0ynByv14RCcX4GHp/kf5fvoLK2iWVbKygrryUv08nae8+zul2LqUnAlCWlxO3zM6gwiw9+NCOkY1wsXA4baQ5bRAc/s/ks1dFaHRnldVUo2gGuOP6t/ExXzPMt4ezhoU/h5mSyNIcNt9fPY0u28OxHuygr1+JFdAGSDHRHfU2jhxdW7OKuV9bijdKKVQ9xdTlslBRmNauacrrTbuST6Ji1lvBziiDKqK5QtAOccUKoU1FJWO+lreP3S+67ZCTds1zcsWAtOyrqkv6eOlJKvAGBVBMoGAgwuEc23585OGK8HpbbklBzp13wl0/3MKJXLldNKsZhtxlay1lDi1i+rSJqddiujvpGFIp2QHgIdax2mkl7z7DFtndeBjeeMYBLx/eJckXy8PikUXHV7KTef8w6okkPSW1JqLleEHHev74yBJHugD8lV/M/JNvM9N/1B9l1NHUC9mShNAiFoh0Q/vSaiF2+tZgX26evm8jUgd1jjI68pjWYs6LNnzVabkJQg2idJrXhoFYRdldAOxrRKxeABrePXIvgATNSSj4uO8oZgwrjmrhue+lLbAJ2PnpRq+bb1igNQqFoB4SbmMxP1V+bVBw+PDnvafJ7XDS2lxEhdDIwh9AeM/XGDncm67TGxGTGGxBA9R4vLrvNiNxqcGvNk579cGfUxL0Ptlbwjf9bxbMf7Yz5HrovpzO0wlYahKL5+DzQeELbGo5D4/Hgfshr/fyJ0C3nFBh2EQy/CPqdBnb1ZxjupK42PVU7UmQbF0Jwakk+cydHtiwdWJjFTgsTicfvR0qZUBZzLMy9Jsqrm4zX3mgahE8zAbVEQGz81fmMum8pEEywe2X1Ptw+PxlO7W+v3u3jaK2bhxdv5u+f7WH5j2dG3KcqMOcPt1eQlebg+qn9I8as23ecS59e0ew5tlfUf2ZXpbEaGqqiLOJxFnlPHNuqzQHpeZCRB+ndtC2vX+B1LlRshTXPw2d/gox8GDobhl0Ig88BV3Ib43QUTslNj3rOqixHsvjn/5xueXzpXdMZ8vPIQspSQkVtEz1yos83EfT+E727pYcIooqaJsvxesmRlpi4zCHCRdlp1DV5jWisjEA9pwaPjxMN2pwORsn10AMFVpRVsqKskhnDiiI6/a3eXdXs+bVnlIDoKrjrYe8nsGMZ7PwAjmyIMVgEF3Z9kS8cHDiWF9jCzhvnuoEzI34thKZa2PE+bF0MW5fAupfBkQ4DZ2iaxdALIDv15VPaCw9fNob/Rik/7bCdfEtwrIieyQ+/x65HL2yVFvHmOq2Ka9/umRw8ESzkvGp3Fa9/vp/zR/cMSR7UTUytrahc5/Ya2gRgtHxtcPuMLO1onyu8uN/xeg/F+aFj0jpZ33olIDorfj8cXhcQCMtg72fgawK7C/pOgZm/gNzeFot8N3DlQKoXpbRsGDlH23xe2PspbFkEWxfBtrcAoc1z+IUw/GIoiN+voyPTLdOJEMFeymbaYw2w2iavZUa4mVdX7+PdzUd45pshfcBwe/28Fuh5UWSRxXz3P9exouwoT14zPuQaaL0Poq4pNFpJbwN7/f99xmv/c5rxXpsPVRsObJ3wBkpW2o5V34mOjBIQnYnje4MCYedyzYQE0GMUTL4ZBs6E/qe1PzOO3QEDpmnb7Ec17WbLYtjyX3jnXm0rHKZpFsMvht4TUi/A2oBVPzuX2iYvM3/zQcjx8MJ97YHj9Z64AuInr6+3PO71BxfaaI2LDp0IaQ8TkijXGo43hLZWNeeXeE1e5Qt+/xHv330W3bNc5AWSFMNDYStqQwXEgeMNPLF0K6CFzx6pbuLA8Qb6hOWbdCSUgOjINJ6AXR9pAmHHMqjaoR3P7glDz9cEwsAZmlO4oyAE9ByjbTN+qgm9rUs0YbHi9/Dxk9rnG36h5ugeMA0cnaOWTlFOWsgT9fCeOWw5XHPSiuY1h6o6N33Dyl00eX0sWLWP66b0CzFRNbh9hq0fQhdiPYpo7uR+VNY28famI0CkOcfdCh8EaI7qhxZtYqGpQdHY4m6GiQkiW5qe/dvlIaVFVu8K9S/85ZPdXF0aLFj9zPIdxutvnV7Cr9/ayr6q+lYLiAa3D4k0KtmeTJSA6Ej4PLB/TVAgHPgcpA+cWVByZlBLKBqWeD3k9k5eP5jyXW2rr4Lt72jCYt0rmqPblQNDztU0iyGzNBNZB2fXoxeyoqySI9WN3P3PdTFrNJ0Mhp2Sw8Vje7FqdxUfbT8KQFW92yilrRevHH3fUjw+SU2jh9vODlaorXN7QwSEzxSpFMw9kFwxsdgQEOG01sSUlebQHNSmHhd98zPpaQoO8FrEpZpLiyxcF9r9zty/AyDb1KVPT3RMRmmSyY+8S02jl7X3zjK0mZOFEhDtGSnh6Lag2Wj3x+CuBWGD3hNh2g81gVB8KjhO7h9Om5DZHcZdo22eRti1POC3WAIb3wCbUxOUwy/SoqK6pT4jOBUIIThzSKHRM7kZZYdSwpI7pmGzCfx+ybXPrGTV7iqqGzyUPvQuADsfuRCbTRhJbhHmobDudeaFODPgiG7y+kMq1ob7YnQTU2vKYYQ7kB+9ckxIwtu3nl9leV1tk9dIrItFXob2P/jgpaMY1lPzX/zpgzJmj+7Z0ikDwUTC8Q+8w+7HTm7inRIQ7Y3aCi3KaGcg2qg60GMpfwCMvVoTCAOmaeGhXRlnumZGG3q+5pA/sEbTLLYsgsU/0rbeE4L5Fj1GdDit6pwRPbimtC93BRoKnWxunTGIg8cbjEXUZhP8fu54Tnv0/ZACd3/+cCe3zggGEYSbasKT38zn9bahbq8/xLyWk+7kyj99Qr3bx5u3nUFTwEHcmrLn60wNgi4a2ytu5rTOk29v4/kVu+KO07+Tayf3M4Tiuv0njPPH691sOljN6YMLmzPtEP65Zh9fK+0bf2CSUAKirfE0wJ5PAmajD+DIV9rx9DwYeBYM/DEMmgn5JW05y/aNzQZ9J2vbrAegYpsWDbVlESx7SNvySzQz1LALoO/UDpGcl+608/hVY9vs/X86e3jEMT3qp8Fkqnn8rS0hAsIepvKEaxBmgaHfz+PzM6p3N1688VRueGE1dU1ePt9zDNDKktcHGvy0xg6/4UBwsTbP8JbpA3nmw+jZ0V/uO2a8ntQ/n2e/WcrEB9+JGNfo9eGwCZx2Gw4LtW/+u9t58ZPdfPSTmRH+m0T58WvrlYDoMiyZp9nRzeGn59yraQm9xoGtc8VUnzSKhmrbmXdBzWHNBLV1Max6Bj59ShO+Q86DYbNh8Lmdwm9xstCjfsy2/GGnaPb280edwtKNRyLs7k0mAVHv9nLXK2uNfd2noJunZgzrQZ+8jJAWoR6f33g/s1O5uTx65VjDjGReoG+eFltAmDWPg8cb6J7l4vazB/PHZWUhWeUNbr/x/QghmDqwO6aALT7doX2mo7VNLRYQAM9/vIubzhzQ4uubgxIQbUnhkPYdftoZyOkJpTdqW1ON5s/ZugS2L4WvXtWyvvufofkshs1Wmloc9EQ1c70id5gJadFXh/ityQRl1hjOeuKDkPwBPRN5dJ+gkA43I9W5fUYiW2sS5aYNLuSG00tId9r57vSBxvHwUNv37z6Ls3+73Ng3W8x0/0pmmgMptdBXXatp9PpCwmbTHHaOm+pM6Z+rtUEHD/x3kxIQXYJTv93WM+hapOUEk/P8Pti/OqBdLIG3fqptPUZqZqihF0CfSZ0y36LF+H0ITwN9nLV4q3YzWByiKM2Pr7KOb/7sEzJwc6mtiQzhpuHjnZxma+AL/5AQE5NZOMwYVsSwnjm8dec0hvQIljcP755X3+Sl3q0txK3J3rbZgeuhFQAAF+VJREFUBPfPGRVxPDwyyuuXDCzKYmcMx7TuO6lp9AYFhMcXItzSnbYQ7Ul3ktcmoVKv1+dPWY0uM0pAKLomNjv0m6pts34FlTu0DO6tS+Dj+fDRbyGrBww9T9MuBs7oWBqezwvHdsHxPVqZFU8DeBu0n57AvvG60XTMdC58vE97Gl5hBw4Auk/ZKg1lObzsggbpovGdyTD6PBg4E4EfGSgirfs4hvcMzVgOdx7Xu33Uu72tMi81h8FF2fTISYstIAJC7Kr//YSPfnI2oAuIUA3C7MzfW6lpXYn24AatiZPOq989jav//CkA724+wuzRvRK+T0tRAkKhAK2Ux2nf17aGY1D2nua32PQmfPl3sKdpQmLYbK24YG7vtp6xhrcJKsugYotWBFHfKsvAHysGX4AzU6ubFbJlagUVc3pqtbH0Y6afv1i8kyacNMg0zhs3gAXrjtIoXTSQRiMuGqQLH3bG2HZypm0DV9fv0LLhgfWZ3fjAM4KP/GMYkTEGyI2YWbjJp86taxCpFxALbzsDm03w+JVjefajnfx95V7Lcbog2FcVLOzX6PEbTnfQzGG6BtHk9XG4WjNPNUeDWLa13HhtvvfCdQeVgFAo2oSMfBhzlbb5PFqUme7o3r4UuAt6jQ/6LXqOTX0IbVOtlhNzdFuoMDi2C2TAjCFsmg+lcJgW/ls0XNtPywZHmBBwpLV4zn9/c5Hx+opxpXzy5RrLce/7J/K+fyK9Z01kdj9g13I2v/c6k0+s4hL7Spj/LHQfpAnegTOM8O20MB9EvdtHdaMnJBEt2Wx5cDY2IQxzU/+CLB66bExUAWFFpIkpqEE8szzoBI9WLdYKvYDj+L55jOydy+Ae2ZSV17L4q8Pc+vfPSXPYmH/thITv11yUgFAoYmF3BsKNz9LqRFVsCfotPngUPngEcosDmsUFrS/90XBMC9Ot2BIqDE7sC46xOTWNp+doTYgVDtWEQcFgLT8kxZT2z2dNIAQ1Xj0mALdPahrXuGt5Zu1g3i0/zGBxgHcv9Wu5PutfgTX/F0gAncAlnpEctPXX/Bc4qWvycqS6id7dUvfZktHzu8HjC6lAa9YgepnKbZRHKWluRW66g5w0B//+/hkAnD6ogLLyWgCWbDgMwGNXjk1Jz3JQAkKhSBwhtIS7HiO0LPbaCk2j2LoE1r4Eq58DV7aWtzLsQi2UNssiKUpKqKsI1QSOBn7WmkpNONK1xb/fVCj6liYECodB9wGa4GojeuVlQEBApDlsRs2oaJid1A0eLyAok8Uw9SKYequphMwHsPMDppe/xEyXj0ZcfOYbjm3n2fRs6E9uj3Ep/mTx6RcITy00dd/75b838K3T+9Po8VOYHVyoM1x26t0+Pt1RGdLTIzwvJBZun590k2ntFxeN5K+f7gkZ0+TxKwGhULQ7sotgwvXa5mmE3R8F+lu8BZvfxChZPmy2ludiFgYNweQr0nI1QTB4ViCHY7hWT6tbv3YZRWUumOew2XjrzumUzFsUdXyTN+iorQ/kMwzpkR0cYHdqYd79T4OZ9zBu3j+ZYtvMmbYNTLNvYPCu+UwDasvy4LVzgyapvJOXMAaw9t5Zhglq8oDu9Oueyd6qev62cg8fbq/AJkTIQj048BnnPruSJwIJj7npjqhtVa1o8vpDvm+rWlTa95uaBwYlIBSKZOBM14oFDpkFFz0Jh9ZpmsW2JfDu/dqYzAJt8R95WUAIBIRBTq8OVQbE7CPQe1VkOO1GOeySgkx2VwbzJJZuPMJ1k/shhMDt9XPuiB48961To96/lkze80/iPf8k8l1OrhvhpHbze1yWs40Juz+CDa9pAwsGB4VFyTStr0mSOWNwASvKtAS38EJ5F47pxf8GKrjWNHpJc9jIMH03ZnOTXk8pK80RkTcSC7fXT6bdrz1QuOvBXccYsZNMmsgUjWTRiH3dYSgo0sK3k0xKBYQQYjbwe8AOPCelfCzKuCuB14BTpZRrhBCzgMcAF+AGfiylfD+Vc1UokoYQ0Hu8ts28B2qOaGG1VuamDki6I/iUrJfVmD60kKUbj/DCDaeyfFsFL36y2xjz4bYK3t50hPNH9cTj88ftkDekRzbby2sZeko2dU0+DsvuvCunw4BrmTBnFJRvNsxRrH1ZM+3pBSwHztC+d5sDhF07brNpP/V9YdN+H/rriP3gdc9f2oOapkIy0pxaVr7pmu72erJowI/A0+TF4XeQJ+rhxAFw11FYc4ApYjOZopFuO3fwNfs++uOn4LgX3lsK7rrg5qkPvK41BAHuWn7XWIcTDzwe/H7eDHdxvYf22TuSgBBC2IGngVnAfmC1EGKhlHJT2Lgc4A7gM9Pho8AlUsqDQojRwFKgY5bmVCg6Uj+OBJg7ua9RvE6vOfSbr43jgtHlzBzeg6kDC0IEBEBZeS3VDfs4eLzRqHQajYcvH8PVf/6UdKcdKbXyHEZimBBwykhtO+174HVrhRp3fqBlyX/8O60EfpJIwzrNA+AW4Baz31wCXwU2YBzwin7xTrjSCTQFto/tmr/KlQWuzMDPbMgshLzAa1cm/914HLctg2tOHx4Yk8Wr6yt5Y+MJ6mQ69aTxyDVTmTysf9I+s5lUahCTgTIp5U4AIcQC4FJgU9i4B9Hk44/1A1LKL03nNwIZQog0KWXi7n+FQpEShpySY2Qa6/b0nHQnl03QnuEyXHZG9c4N6Zew6WA1T3ylhWw647RQ1au9DizMYldlPbVNXjw+aV3q2+GC/qdr28yfaU20qgKhv1JqwkL6tcx56Q/uS79WBdh8zG86Z4yJft1H2w6zfGs5dvzYkAj8nDq0DzNHl4Arm2q/i+++soV6mUYd6TTINIb170WVx8m/fzAzrlnx8IlG7vrwPbplOLnmtPOM43sPbuVTf5mxf0AWpMS8BqkVEH0AU2we+4Ep5gFCiIlAXynlIiHEj7HmSuALK+EghLgFTZDTr1+/pExaoVDE55rSvjy6ZEvUBjbj++aFCIjymmCPiHhd4aYM6M7PLxzBtZP78u2/rKHJ68fj9+NKpDd3ejfNxHQSGD/ewzfufxvQenb4Jfyk/zBmThoMaCmAn74cGl00MquA+sq6hHxOWw5r3194R8HZo3vy1LKggKh3J09jCqfNQiSEEDbgSeDuGGNGoWkX37U6L6V8RkpZKqUsLSoqSs1EFQpFBLdMH8imB84PCfc0c98lo3j91tP58pezAIwsYojf9MdmE9w8fSA56U7SHDYaPT6k5KTUHmoO5hwQPcs5I0a46fCeOaQ77Ubl2njcv3AjAN8JK8w3uk83LhwTbELU0EEFxAHAHIdWHDimkwOMBj4QQuwGpgILhRClAEKIYuAN4JtSyh0oFIp2gxAiZm8Gl8PGpP755Ge5yMt0hpSkaE5XOKfdRl2gdlFrusmlGr0ceXg+woJbphqvnXYbWw9Xs+toXUiV12jokWBnj+gRce7/fX0SZQ9fAHRcDWI1MEQIMUAI4QKuBRbqJ6WUJ6SUhVLKEillCbASmBOIYsoDFgHzpJQrUjhHhUKRYsJrDzkdiYf0uuw2dgSK5sXzXbQHwkuVT+of7PzosAu2HdGyoD/YWhHzPnWmgn72KOYoh92Gy27rmAJCSukFbkOLQNoMvCql3CiEeEAIES8e6zZgMHCvEGJtYIsUowqFot3jDWtBatVtLRr7jgXzKdqzBqGTnRaasGb+rOaOdo44wm57oJwGaMUKo5HhstMQ43xrSWkehJRyMbA47Ni9UcbOML1+CHgolXNTKBRtQ3NKTZj7V7dHAfHWndN4ZPEWPtymaQThlWjN/SvMvgcZxw1xxOSzGVyUE3VcZqCcR6pof9+4QqHo1DQ1Q0DkZQYX3PZoYhreM5d80xyz02I/c+tF93Qh+ea6g/xt5R72VdWzZneVETasO57fv/ss+hVEb0+a4bJT70mdgFClNhQKRUoZdkoOW48Ei/k1eRIXED1ygplo7VGDgFBtICstehTTD2cN5ZRcLepLN5394GUt5WtgYRY7j2q+lqevm2iYlcK764WT6bJ32CgmhUKh4L5LRobsN3oTX9DMPahzUtgPojWYXSpWYa6XjdeaSxVku4wckPnvbg8ZowsHgFfW7DMW/Yw4TZIynQ7qU+iDUAJCoVCklP6FWqvWH84aCkBJQeKtW80L5MCi7Bgj2w5zZ7d0iwX9rllDGdU7l/NH9aR7lpZYOLa4W9T7NQQ66AFkxinjnZFiDaJ9imSFQtFp6JOXwa5HL0QIwTkjehhlsBPhrnOHGv0PYplv2pLZo4NJa1YaRP+CLBbdPs3YH9Ijmz6mBkLhNHh81Lt9uOy2uMmBmS47B44rE5NCoejA6NE8o3p3I82R+EKfnxUs5ZEVIzGvvZCIn6S2ycuyreW8GCh4GM6GA9U0uL1xzUug+SjKymuR8cKiWogSEAqFol3z8s1TuXxCHzITWDA7AodONNLo8XP/m+F1S4McrXUn5HPRE+4Wf3U4afMzowSEQqFo15w2qIDfXTM+JKegIzNtiHVfkLsDPhqARV8dYugp0fMfdEb00sbUNHrijGwZSkAoFApFKxnRK3aPCzNXTLRubXPh2F4hrVj7x8h/0Hn4sjGA1ooiFSgBoVAoFK3kje+dzoZfnZ/Q2Gg+mNx0JyN7BwWNLQGNSc/cXrj2YELv3VyUgFAoFP+/vbsPlqqu4zj+/ggKqSAXJQIfkVCHDAEJydQBY3gwFE1NzRlJnSmfmszJ1CyypgcVs6a0nFREHENz8oEcHUXNaBhAAeGCj4DRAyEoMmJqivjtj9934XA5e6/LvWd3L/f7mtnZc3+7e/Z7fufufvf8zu73F1qp666dWvwVdUn2tx0lP574GXp168KFI/tvadvwTssVX7t17cyEQX0YcfDeHz/YCtT/1wJCCGEn0vTbSXedP5xjB6T5bA77VHcuGtmf3z69kk0ftTxwtMsu4qavDi0kTogjiBBCqKrsbyCmnDZoS3IomTAo/fJ6yP7FTCNaiTiCCCGEKuqbSRAnHtF3u9sH9u3O7MtH0bdH1+1uq7ZIECGEUEW7dtqFyRMGMviAHtvNQFfSXAXXaooEEUIIVXZek3mm61WcgwghhJArEkQIIYRckSBCCCHkigQRQgghVySIEEIIuSJBhBBCyBUJIoQQQq5IECGEEHKpqKnqqk3S68A/WrGKfYA32iicIkR8rRPxtU7E1zr1HN+BZtYr74adJkG0lqQFZjas1nGUE/G1TsTXOhFf69R7fOXEEFMIIYRckSBCCCHkigSx1e9rHUALIr7WifhaJ+JrnXqPL1ecgwghhJArjiBCCCHkigQRQgghV4dPEJLGSXpZ0gpJV9Yohv0l/UXSC5Kel/Qtb79G0mpJi/1yQuYxV3nML0saW4UYV0la6nEs8LaekmZJWu7XDd4uSb/2+BolFTerenq+QzN9tFjSRkmX1rL/JE2VtE7Sskxbxf0laZLff7mkSQXHN0XSSx7DA5J6ePtBkt7L9OMtmccc6f8XK3wbVGB8Fe/Pol7fZeK7NxPbKkmLvb3q/ddmzKzDXoBOwErgYGA3YAkwsAZx9AGG+nI34BVgIHAN8J2c+w/0WLsA/XwbOhUc4ypgnyZt1wNX+vKVwHW+fALwKCBgBDC/yvv0NeDAWvYfcBwwFFi2o/0F9ARe9esGX24oML4xQGdfvi4T30HZ+zVZzzMes3wbxhcYX0X7s8jXd158TW7/BTC5Vv3XVpeOfgQxHFhhZq+a2QfAPcDEagdhZmvMbJEvvw28COzbzEMmAveY2ftm9ndgBWlbqm0icKcv3wmcnGmfbsk8oIekPlWK6YvASjNr7lf1hfefmc0G3sx53kr6aywwy8zeNLMNwCxgXFHxmdnjZvah/zkP2K+5dXiM3c1snqV3u+mZbWrz+JpRbn8W9vpuLj4/CvgKMKO5dRTZf22loyeIfYF/Zf7+N82/MRdO0kHAEGC+N13ih/xTS0MS1CZuAx6XtFDS172tt5mt8eXXgN41jK/kTLZ9YdZL/0Hl/VXLfjyP9Im2pJ+k5yT9VdKx3ravx1TN+CrZn7Xqv2OBtWa2PNNWL/1XkY6eIOqKpD2BPwGXmtlG4HdAf2AwsIZ02Forx5jZUGA8cLGk47I3+iegmn5nWtJuwEnAfd5UT/23jXror3IkXQ18CNztTWuAA8xsCHAZ8AdJ3WsQWt3uzybOYtsPKfXSfxXr6AliNbB/5u/9vK3qJO1KSg53m9n9AGa21sw2m9lHwK1sHQapetxmttqv1wEPeCxrS0NHfr2uVvG58cAiM1vrsdZN/7lK+6vqcUr6GjABONuTGD50s96XF5LG9Q/xWLLDUIXGtwP7sxb91xn4MnBvJu666L8d0dETxLPAAEn9/NPnmcDMagfhY5a3Ay+a2Y2Z9uy4/SlA6RsTM4EzJXWR1A8YQDrZVVR8e0jqVlomncxc5nGUvlkzCXgoE985/u2cEcBbmaGVIm3zya1e+i+j0v56DBgjqcGHU8Z4WyEkjQO+C5xkZu9m2ntJ6uTLB5P661WPcaOkEf4/fE5mm4qIr9L9WYvX92jgJTPbMnRUL/23Q2p9lrzWF9I3SF4hZfWraxTDMaThhkZgsV9OAO4Clnr7TKBP5jFXe8wvU/A3H0jfAlnil+dL/QTsDTwJLAeeAHp6u4CbPb6lwLAq9OEewHpgr0xbzfqPlKjWAJtIY8vn70h/kc4FrPDLuQXHt4I0Zl/6H7zF73uq7/fFwCLgxMx6hpHeqFcCN+HVGQqKr+L9WdTrOy8+b58GXNDkvlXvv7a6RKmNEEIIuTr6EFMIIYQyIkGEEELIFQkihBBCrkgQIYQQckWCCCGEkCsSRGhXJG32iphLJC2SdHQL9+8h6aKPsd6nJbW7SeWLJGmapNNqHUeonUgQob15z8wGm9kRwFXAz1u4fw+gxQRRK/7L2xDqUiSI0J51BzZAqmMl6Uk/qlgqqVS181qgvx91TPH7XuH3WSLp2sz6Tpf0jKRXSgXVJHVSmifhWS8S9w1v7yNptq93WaYA2xZKcwJc78/1jKRPe/s0SbdImg9crzRPxIO+/nmSBmW26Q5/fKOkU719jKS5vq33eQ0vJF2rNKdIo6QbvO10j2+JpNktbJMk3aQ0f8ITwCfbcmeF9ic+vYT25hNKE7F0Jc2jcby3/w84xcw2StoHmCdpJmnehcPNbDCApPGkks9Hmdm7knpm1t3ZzIYrTUTzQ1LZhPNJpS8+J6kLMEfS46R6O4+Z2U+9jMLuZeJ9y8w+K+kc4FekOkeQ6u4cbWabJf0GeM7MTpZ0PKns82DgB6XHe+wNvm3fB0ab2TuSrgAuk3QzqfzEYWZm8sl+gMnAWDNbnWkrt01DgENJ8yv0Bl4Apn6svRJ2SpEgQnvzXubN/vPAdEmHk8pV/EypyuxHpLLJvXMePxq4w7zWkJlla/rf79cLSZO8QKp/NCgzFr8XqZbOs8BUpSKLD5rZ4jLxzshc/zLTfp+ZbfblY0jlGDCzpyTtrVTtczSpfhB+2wZJE0hv4HNS+R52A+YCb5GS5O2SHgYe9ofNAaZJ+mNm+8pt03HADI/rP5KeKrNNoYOIBBHaLTOb65+oe5Fq7vQCjjSzTZJWkY4yKvG+X29m62tDwDfNbLsieZ6MvkR6A77RzKbnhVlm+Z0KY9vytKRJhM7KiWc4acKk04BLgOPN7AJJR3mcCyUdWW6blJnCMwSIcxChHZN0GGlayfWkT8HrPDmMIk05CvA2aRrXklnAuZJ293Vkh5jyPAZc6EcKSDpEqbrtgaRJYW4FbiNNP5nnjMz13DL3+Rtwtq9/JPCGpflAZgEXZ7a3gTTT2xcy5zP28Jj2JBUqfAT4NnCE397fzOab2WTgdVL569xtAmYDZ/g5ij7AqBb6Juzk4ggitDelcxCQPglP8nH8u4E/S1oKLABeAjCz9ZLmKE0u/6iZXS5pMLBA0gfAI8D3mnm+20jDTYuUxnReJ00LORK4XNIm4L+kUs15GiQ1ko5OtvvU764hDVc1Au+ytST4T4CbPfbNwI/M7H6lORtm+PkDSOck3gYektTV++Uyv22KpAHe9iSpIm9jmW16gHRO5wXgn5RPaKGDiGquIRTEh7mGmdkbtY4lhB0RQ0whhBByxRFECCGEXHEEEUIIIVckiBBCCLkiQYQQQsgVCSKEEEKuSBAhhBBy/R9mgKVfcEbqAQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.recorder.plot_losses()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "d = data_clas.train_ds[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Category 0, tensor(0), tensor([0.8236, 0.1764]))"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.predict(\"aaa\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "d = TextClasDataBunch.from_csv(dir_pth, 'train.csv', test = 'test.csv',vocab=data_lm.train_ds.vocab, bs=32, text_cols=3, label_cols=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3263"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(d.test_ds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = test_data.iloc[1, 3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Heard about #earthquake is different cities, stay safe everyone.'"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[   2,    5,  413,   75,   13,  320,   26, 1635, 1925, 3135,  749, 1391,\n",
       "           283,   15]], device='cuda:0'), tensor([0], device='cuda:0'))"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.data.one_item(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "def p(x):\n",
    "    return learn.predict(x)[1].item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data['label'] = test_data['text'].apply(p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>keyword</th>\n",
       "      <th>location</th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Just happened a terrible car crash</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Heard about #earthquake is different cities, s...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>there is a forest fire at spot pond, geese are...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Apocalypse lighting. #Spokane #wildfires</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>11</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Typhoon Soudelor kills 28 in China and Taiwan</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id keyword location                                               text  \\\n",
       "0   0     NaN      NaN                 Just happened a terrible car crash   \n",
       "1   2     NaN      NaN  Heard about #earthquake is different cities, s...   \n",
       "2   3     NaN      NaN  there is a forest fire at spot pond, geese are...   \n",
       "3   9     NaN      NaN           Apocalypse lighting. #Spokane #wildfires   \n",
       "4  11     NaN      NaN      Typhoon Soudelor kills 28 in China and Taiwan   \n",
       "\n",
       "   label  \n",
       "0      1  \n",
       "1      1  \n",
       "2      1  \n",
       "3      1  \n",
       "4      1  "
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub = test_data[['id', 'label']]\n",
    "sub.columns = ['id', 'target']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub.to_csv('./submissionb.csv', index = None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/usr/bin/sh: kaggle: command not found\n"
     ]
    }
   ],
   "source": [
    "!kaggle "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "liangjiaxi",
   "language": "python",
   "name": "liangjiaxi"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
